https://furukama.com/Recent content on Furukama's BlogHugo -- gohugo.ioenben@furukama.com (Benedikt Koehler)ben@furukama.com (Benedikt Koehler)https://furukama.com/llm-fu/Fri, 05 Sep 2025 00:00:00 +0200ben@furukama.com (Benedikt Koehler)https://furukama.com/llm-fu/<p>Fu-Benchmark is a meta-benchmark of the most influential evaluation suites used to measure and rank large language models. Use the search box to filter by name, topic, or model.</p> <div class="llm-fu"> <style> .llm-fu table { font-size: 0.85em; } .llm-fu th, .llm-fu td { padding: 3px 5px; } .llm-fu thead th { position: sticky; top: 0; background: inherit; } .llm-fu .logo { width: 14px; height: 14px; margin-right: 0; vertical-align: text-bottom; opacity: 0.95; } .llm-fu .avatar { display: inline-flex; align-items: center; justify-content: center; width: 12px; height: 12px; margin-right: 6px; border-radius: 50%; border: 1px solid var(--color-light); font-size: 8px; line-height: 1; opacity: 0.85; } .llm-fu .stars { color: #f5c518; white-space: nowrap; } .llm-fu .stars .empty { opacity: .35; } .llm-fu .flag { margin-left: 2px; font-size: 0.95em; line-height: 1; display: inline-block; } .llm-fu .flag img { width:14px; height:14px; vertical-align:middle; } .llm-fu .brand { display: inline-flex; align-items: center; gap: 0; margin-right: 6px; } .llm-fu td.leader-cell .leader-link { display: inline-flex; align-items: center; gap: .4rem; color: inherit; text-decoration: none; } .llm-fu td.leader-cell .leader-link:hover { text-decoration: underline; } .llm-fu td.leader-cell .brand { gap: .3rem; } .llm-fu td.leader-cell .leader-name { white-space: nowrap; } .llm-fu .lb-icon { display:inline-flex; align-items:center; justify-content:center; width:14px; height:14px; margin-left:6px; opacity:.8; vertical-align: text-bottom; } .llm-fu .lb-icon svg { width:100%; height:100%; } .llm-fu .doc-icon { display:inline-flex; align-items:center; justify-content:center; width:14px; height:14px; margin-left:6px; opacity:.8; vertical-align: text-bottom; } .llm-fu .doc-icon svg { width:100%; height:100%; } .llm-fu .data-icon { display:inline-flex; align-items:center; justify-content:center; width:14px; height:14px; margin-left:6px; opacity:.8; vertical-align: text-bottom; } .llm-fu .data-icon svg { width:100%; height:100%; } .llm-fu .gh-icon { display:inline-flex; align-items:center; justify-content:center; width:14px; height:14px; margin-left:6px; opacity:.8; vertical-align: text-bottom; } .llm-fu .gh-icon svg { width:100%; height:100%; } .llm-fu td.desc { font-size: 0.9em; } .llm-fu .score-symbol { color: #9ca3af; margin-left: 6px; font-weight: 600; } .llm-fu-histograms { display:grid; grid-template-columns:repeat(auto-fit,minmax(320px,1fr)); gap:.9rem; margin:1.1rem 0 .2rem; } .llm-fu-histogram { background:rgba(15,23,42,.08); border:1px solid var(--color-light); border-radius:8px; padding:.45rem .85rem .1rem; } .llm-fu-histogram-head { display:flex; align-items:center; justify-content:space-between; gap:.75rem; margin-bottom:.35rem; } .llm-fu-histogram h3 { margin:0; font-size:1.1rem; } .llm-fu-log-toggle { display:inline-flex; align-items:center; gap:.35rem; font-size:.9rem; opacity:.85; white-space:nowrap; } .llm-fu-log-toggle input { margin:0; } .llm-fu-legend { display:flex; align-items:center; gap:.75rem; font-size:.85rem; flex-wrap:wrap; } .llm-fu-legend-item { display:inline-flex; align-items:center; gap:.35rem; } .llm-fu-legend-swatch { width:12px; height:12px; border-radius:3px; display:inline-block; } .llm-fu-histogram-footer { display:flex; align-items:center; justify-content:space-between; margin-top:-.5rem; font-size:.8rem; color:var(--color-muted); } .llm-fu-histogram-chart { width:100%; height:300px; position:relative; } .llm-fu img.logo[data-company="nousresearch"], .llm-fu img.logo[data-company="liquidai"] { filter: brightness(0) invert(1) !important; } @media (prefers-color-scheme: dark) { .llm-fu .logo { filter: none; } .llm-fu .logo.no-invert { filter: none !important; } } </style> <div class="llm-fu__controls" style="display:flex;gap:.75rem;align-items:center;justify-content:space-between;flex-wrap:wrap;margin:1rem 0"> <div style="display:flex;gap:.75rem;align-items:center;flex:1"> <select id="llm-fu-open-filter" style="padding:.45rem .6rem;border:1px solid var(--color-light);background:transparent;color:inherit;border-radius:6px"> <option value="all">All models</option> <option value="open">Open Weights models</option> <option value="openplus">Open Source models</option> <option value="commercial">Commercial models</option> </select> <select id="llm-fu-modality-filter" style="padding:.45rem .6rem;border:1px solid var(--color-light);background:transparent;color:inherit;border-radius:6px"> <option value="all">All modalities</option> <option value="text">Text</option> <option value="image">Image</option> </select> <select id="llm-fu-size-filter" style="padding:.45rem .6rem;border:1px solid var(--color-light);background:transparent;color:inherit;border-radius:6px"> <option value="all">All sizes</option> <option value="XS">XS (&lt;2B)</option> <option value="S">S (2–7B)</option> <option value="M">M (7–30B)</option> <option value="L">L (30–70B)</option> <option value="XL">XL (70B+)</option> </select> <select id="llm-fu-year-filter" style="padding:.45rem .6rem;border:1px solid var(--color-light);background:transparent;color:inherit;border-radius:6px;min-width:140px"> <option value="all">All years</option> <option value="2026">2026</option> <option value="2025">2025</option> <option value="2024">2024</option> <option value="2023">2023</option> <option value="2022">2022</option> </select> <select id="llm-fu-country-filter" style="padding:.45rem .6rem;border:1px solid var(--color-light);background:transparent;color:inherit;border-radius:6px;min-width:210px"> <option value="all">All countries / regions</option> <option value="" disabled>──────────</option> <option value="" disabled>Aggregates</option> <option value="EU">European Union (EU)</option> <option value="EUROPE">Europe (EU + UK, CH, NO, IS)</option> <option value="EMEA">EMEA (Europe + Middle East)</option> <option value="" disabled>──────────</option> <option value="" disabled>Individual Countries</option> </select> <input id="llm-fu-search" type="search" placeholder="Search name, topic, model…" style="flex:1;min-width:220px;padding:.5rem .75rem;border:1px solid var(--color-light);background:transparent;color:inherit;border-radius:6px"/> <a href="#llm-fu-table" style="white-space:nowrap">Benchmarks</a> </div> </div> <div id="llm-fu-topn-toggle" style="display:flex;gap:.35rem;margin:0 0 .5rem"> <button class="topn-btn active" data-topn="10">Top 10</button> <button class="topn-btn" data-topn="25">Top 25</button> </div> <div id="llm-fu-mini" class="llm-fu__mini" style="display:flex;gap:1rem;flex-wrap:nowrap;overflow-x:auto;margin:0 0 1.25rem"></div> <div id="llm-fu-mini-scores" class="llm-fu__mini" style="display:flex;gap:1rem;flex-wrap:nowrap;overflow-x:auto;margin:0 0 1.25rem"></div> <div style="overflow:auto;max-width:100%"> <table id="llm-fu-table" style="min-width:780px"> <thead style="position:sticky;top:0"> <tr> <th data-type="num" style="white-space:nowrap">#</th> <th data-type="text" style="white-space:nowrap">Name</th> <th data-type="text">Topic</th> <th data-type="text">Description</th> <th data-type="num" style="white-space:nowrap">Relevance</th> <th data-type="num" style="white-space:nowrap">GitHub ★</th> <th data-type="text" style="white-space:nowrap">Leader</th> <th data-type="num" style="white-space:nowrap">Top %</th> </tr> </thead> <tbody> <tr data-bench="aa-index" data-open-source="0"> <td style="text-align:center" data-sort="1">1</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aa-index" class="llm-fu-bench-link" data-bench-id="aa-index" title="View models and scores"><strong>AA-Index</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multi-domain QA</td> <td class="desc">Comprehensive QA index across diverse domains.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4</span> </a> </td> <td data-sort="73.2"> 73.2% </td> </tr> <tr data-bench="aa-lcr" data-open-source="0"> <td style="text-align:center" data-sort="2">2</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aa-lcr" class="llm-fu-bench-link" data-bench-id="aa-lcr" title="View models and scores"><strong>AA-LCR</strong></a> <span class="score-symbol" title="98 scores" aria-label="98 scores">O</span> <a href="https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://huggingface.co/datasets/ArtificialAnalysis/AA-LCR" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context reasoning</td> <td class="desc">A challenging benchmark measuring language models&#39; ability to extract, reason about, and synthesize information from long-form documents ranging from 10k to 100k tokens.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="76.0"> 76.0% </td> </tr> <tr data-bench="aa-omniscience" data-open-source="0"> <td style="text-align:center" data-sort="3">3</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aa-omniscience" class="llm-fu-bench-link" data-bench-id="aa-omniscience" title="View models and scores"><strong>AA-Omniscience</strong></a> <span class="score-symbol" title="25 scores" aria-label="25 scores">O</span> <a href="https://artificialanalysis.ai/evaluations/omniscience" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://artificialanalysis.ai/evaluations/omniscience" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Knowledge and hallucination</td> <td class="desc">Benchmark measuring factual recall and hallucination across economically relevant domains.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro-preview"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro Preview</span> </a> </td> <td data-sort="13.0"> 13.0% </td> </tr> <tr data-bench="acebench" data-open-source="1"> <td style="text-align:center" data-sort="4">4</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=acebench" class="llm-fu-bench-link" data-bench-id="acebench" title="View models and scores"><strong>AceBench</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Industry QA</td> <td class="desc">Industry-focused benchmark assessing domain QA and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2</span> </a> </td> <td data-sort="82.2"> 82.2% </td> </tr> <tr data-bench="acp-bench-bool" data-open-source="1"> <td style="text-align:center" data-sort="5">5</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=acp-bench-bool" class="llm-fu-bench-link" data-bench-id="acp-bench-bool" title="View models and scores"><strong>ACP-Bench Bool</strong></a> <span class="score-symbol" title="23 scores" aria-label="23 scores">O</span> <a href="https://ibm.github.io/ACPBench/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2410.05669" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/ibm/ACPBench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety evaluation (boolean)</td> <td class="desc">Safety and behavior evaluation with yes/no questions.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-32b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-32B</span> </a> </td> <td data-sort="85.1"> 85.1% </td> </tr> <tr data-bench="acp-bench-mcq" data-open-source="1"> <td style="text-align:center" data-sort="6">6</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=acp-bench-mcq" class="llm-fu-bench-link" data-bench-id="acp-bench-mcq" title="View models and scores"><strong>ACP-Bench MCQ</strong></a> <span class="score-symbol" title="23 scores" aria-label="23 scores">O</span> <a href="https://ibm.github.io/ACPBench/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2410.05669" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/ibm/ACPBench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety evaluation (MCQ)</td> <td class="desc">Safety and behavior evaluation with multiple-choice questions.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B</span> </a> </td> <td data-sort="82.1"> 82.1% </td> </tr> <tr data-bench="aethercode" data-open-source="0"> <td style="text-align:center" data-sort="7">7</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aethercode" class="llm-fu-bench-link" data-bench-id="aethercode" title="View models and scores"><strong>AetherCode</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Code generation</td> <td class="desc">Code generation benchmark for diverse coding tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="73.8"> 73.8% </td> </tr> <tr data-bench="agentcompany" data-open-source="0"> <td style="text-align:center" data-sort="8">8</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=agentcompany" class="llm-fu-bench-link" data-bench-id="agentcompany" title="View models and scores"><strong>AgentCompany</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Agent reasoning</td> <td class="desc">Company-level agent reasoning and decision-making benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-4.5-sonnet"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5</span> </a> </td> <td data-sort="41.0"> 41.0% </td> </tr> <tr data-bench="agentdojo" data-open-source="0"> <td style="text-align:center" data-sort="9">9</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=agentdojo" class="llm-fu-bench-link" data-bench-id="agentdojo" title="View models and scores"><strong>AgentDojo</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> <a href="https://agentdojo.spylab.ai/results/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://agentdojo.spylab.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2406.13352" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/ethz-spylab/agentdojo" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Agent evaluation</td> <td class="desc">Interactive evaluation suite for autonomous agents across tools and tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 3.7" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-3.7-sonnet"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude 3.7 Sonnet</span> </a> </td> <td data-sort="88.7"> 88.7% </td> </tr> <tr data-bench="agentic-coding" data-open-source="0"> <td style="text-align:center" data-sort="10">10</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=agentic-coding" class="llm-fu-bench-link" data-bench-id="agentic-coding" title="View models and scores"><strong>Agentic Coding</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> </td> <td>Agentic coding</td> <td class="desc">Agentic coding benchmark for autonomous software tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-flash-preview"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Flash Preview</span> </a> </td> <td data-sort="53.8"> 53.8% </td> </tr> <tr data-bench="agieval" data-open-source="1"> <td style="text-align:center" data-sort="11">11</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=agieval" class="llm-fu-bench-link" data-bench-id="agieval" title="View models and scores"><strong>AGIEval (English)</strong></a> <span class="score-symbol" title="27 scores" aria-label="27 scores">O</span> </td> <td>Exams</td> <td class="desc">English subset of AGIEval; academic and professional exam questions.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-32b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL 32B Thinking</span> </a> </td> <td data-sort="92.2"> 92.2% </td> </tr> <tr data-bench="agieval-lsat-ar" data-open-source="1"> <td style="text-align:center" data-sort="12">12</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=agieval-lsat-ar" class="llm-fu-bench-link" data-bench-id="agieval-lsat-ar" title="View models and scores"><strong>AGIEval LSAT-AR</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Law exam reasoning</td> <td class="desc">LSAT Analytical Reasoning subset from AGIEval benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-32b-base"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5 32B Base</span> </a> </td> <td data-sort="30.4"> 30.4% </td> </tr> <tr data-bench="ai2d" data-open-source="0"> <td style="text-align:center" data-sort="13">13</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ai2d" class="llm-fu-bench-link" data-bench-id="ai2d" title="View models and scores"><strong>AI2D</strong></a> <span class="score-symbol" title="144 scores" aria-label="144 scores">O</span> </td> <td>Diagram understanding (VQA)</td> <td class="desc">Visual question answering over science and diagram images.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="98.7"> 98.7% </td> </tr> <tr data-bench="aicodeking-non-agentic" data-open-source="0"> <td style="text-align:center" data-sort="14">14</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aicodeking-non-agentic" class="llm-fu-bench-link" data-bench-id="aicodeking-non-agentic" title="View models and scores"><strong>AICodeKing Non-Agentic</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> </td> <td>Code generation (non-agentic)</td> <td class="desc">Non-agentic code generation benchmark from AICodeKing.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="aider-code-editing" data-open-source="0"> <td style="text-align:center" data-sort="15">15</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aider-code-editing" class="llm-fu-bench-link" data-bench-id="aider-code-editing" title="View models and scores"><strong>Aider Code Editing</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://aider.chat/docs/leaderboards/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code editing</td> <td class="desc">Measures interactive code editing quality within the Aider assistant workflow.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="89.8"> 89.8% </td> </tr> <tr data-bench="aider-polyglot" data-open-source="0"> <td style="text-align:center" data-sort="16">16</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aider-polyglot" class="llm-fu-bench-link" data-bench-id="aider-polyglot" title="View models and scores"><strong>Aider-Polyglot</strong></a> <span class="score-symbol" title="118 scores" aria-label="118 scores">O</span> <a href="https://aider.chat/docs/leaderboards/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://aider.chat/docs/leaderboards/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code assistant eval</td> <td class="desc">Aider polyglot coding leaderboard.</td> <td data-sort="2"> <span class="stars" title="Usage: 6/39; GitHub>100: no; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro-preview"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro Preview</span> </a> </td> <td data-sort="92.9"> 92.9% </td> </tr> <tr data-bench="aider-polyglot-diff" data-open-source="0"> <td style="text-align:center" data-sort="17">17</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aider-polyglot-diff" class="llm-fu-bench-link" data-bench-id="aider-polyglot-diff" title="View models and scores"><strong>Aider-Polyglot (Diff)</strong></a> <span class="score-symbol" title="19 scores" aria-label="19 scores">O</span> <a href="https://aider.chat/docs/leaderboards/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://aider.chat/docs/leaderboards/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code assistant eval</td> <td class="desc">Aider polyglot leaderboard using diff mode (pass@2).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro-preview"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro Preview</span> </a> </td> <td data-sort="91.9"> 91.9% </td> </tr> <tr data-bench="aime-2024" data-open-source="1"> <td style="text-align:center" data-sort="18">18</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aime-2024" class="llm-fu-bench-link" data-bench-id="aime-2024" title="View models and scores"><strong>AIME 2024</strong></a> <span class="score-symbol" title="203 scores" aria-label="203 scores">O</span> <a href="https://artofproblemsolving.com/wiki/index.php/2024_AIME_I_Problems" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">American Invitational Mathematics Examination 2024 problems.</td> <td data-sort="2"> <span class="stars" title="Usage: 9/39; GitHub>100: no; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-OSS" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-oss-120b"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-OSS 120B</span> </a> </td> <td data-sort="96.6"> 96.6% </td> </tr> <tr data-bench="aime-2024-ko" data-open-source="1"> <td style="text-align:center" data-sort="19">19</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aime-2024-ko" class="llm-fu-bench-link" data-bench-id="aime-2024-ko" title="View models and scores"><strong>AIME 2024-Ko</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math (competition, Korean)</td> <td class="desc">Korean translation of AIME 2024 problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Thinking-2507</span> </a> </td> <td data-sort="80.3"> 80.3% </td> </tr> <tr data-bench="aime-2025" data-open-source="0"> <td style="text-align:center" data-sort="20">20</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aime-2025" class="llm-fu-bench-link" data-bench-id="aime-2025" title="View models and scores"><strong>AIME 2025</strong></a> <span class="score-symbol" title="455 scores" aria-label="455 scores">O</span> <a href="https://www.kaggle.com/benchmarks/open-benchmarks/aime-2025" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://artofproblemsolving.com/wiki/index.php/AIME_Problems_and_Solutions" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">American Invitational Mathematics Examination 2025 problems.</td> <td data-sort="3"> <span class="stars" title="Usage: 12/39; GitHub>100: no; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-4.5-sonnet"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="aime-2026-i" data-open-source="0"> <td style="text-align:center" data-sort="21">21</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=aime-2026-i" class="llm-fu-bench-link" data-bench-id="aime-2026-i" title="View models and scores"><strong>AIME 2026 I</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Math (competition)</td> <td class="desc">American Invitational Mathematics Examination 2026 I problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="97.5"> 97.5% </td> </tr> <tr data-bench="ainstein-swe-bench" data-open-source="0"> <td style="text-align:center" data-sort="22">22</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ainstein-swe-bench" class="llm-fu-bench-link" data-bench-id="ainstein-swe-bench" title="View models and scores"><strong>AInstein-SWE-Bench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Agentic coding</td> <td class="desc">AInstein agent coding benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="42.8"> 42.8% </td> </tr> <tr data-bench="alignbench" data-open-source="1"> <td style="text-align:center" data-sort="23">23</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=alignbench" class="llm-fu-bench-link" data-bench-id="alignbench" title="View models and scores"><strong>AlignBench</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Alignment and instruction following</td> <td class="desc">Benchmark for instruction-following quality and alignment behavior.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="JoyAI-LLM" data-model-version="JoyAI-LLM" data-provider="jd"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=joyai-llm-flash"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">JoyAI-LLM Flash</span> </a> </td> <td data-sort="8.2"> 8.2% </td> </tr> <tr data-bench="all-angles-bench" data-open-source="1"> <td style="text-align:center" data-sort="24">24</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=all-angles-bench" class="llm-fu-bench-link" data-bench-id="all-angles-bench" title="View models and scores"><strong>All-Angles Bench</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Spatial perception</td> <td class="desc">All-Angles benchmark for spatial recognition and 3D perception.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step3-VL" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step3-vl-10b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step3-VL-10B</span> </a> </td> <td data-sort="57.2"> 57.2% </td> </tr> <tr data-bench="alpacaeval" data-open-source="1"> <td style="text-align:center" data-sort="25">25</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=alpacaeval" class="llm-fu-bench-link" data-bench-id="alpacaeval" title="View models and scores"><strong>AlpacaEval</strong></a> <span class="score-symbol" title="84 scores" aria-label="84 scores">O</span> <a href="https://github.com/tatsu-lab/alpaca_eval" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Instruction following</td> <td class="desc">Automatic eval using GPT-4 as a judge.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1849">1849</td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="99.4"> 99.4% </td> </tr> <tr data-bench="alpacaeval-2.0" data-open-source="1"> <td style="text-align:center" data-sort="26">26</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=alpacaeval-2.0" class="llm-fu-bench-link" data-bench-id="alpacaeval-2.0" title="View models and scores"><strong>AlpacaEval 2.0</strong></a> <span class="score-symbol" title="46 scores" aria-label="46 scores">O</span> </td> <td>Instruction following</td> <td class="desc">Updated AlpacaEval with improved prompts and judging.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek R1" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-r1"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek R1</span> </a> </td> <td data-sort="87.6"> 87.6% </td> </tr> <tr data-bench="amc-23" data-open-source="1"> <td style="text-align:center" data-sort="27">27</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=amc-23" class="llm-fu-bench-link" data-bench-id="amc-23" title="View models and scores"><strong>AMC-23</strong></a> <span class="score-symbol" title="45 scores" aria-label="45 scores">O</span> <a href="https://www.maa.org/math-competitions/amc-1012" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">American Mathematics Competition 2023 evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="QwQ" data-model-version="QwQ" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwq-32b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">QwQ-32B</span> </a> </td> <td data-sort="98.5"> 98.5% </td> </tr> <tr data-bench="amobench" data-open-source="0"> <td style="text-align:center" data-sort="28">28</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=amobench" class="llm-fu-bench-link" data-bench-id="amobench" title="View models and scores"><strong>AMO-Bench</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> </td> <td>Math (competition)</td> <td class="desc">Advanced math olympiad-style benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="72.5"> 72.5% </td> </tr> <tr data-bench="amobench-zh" data-open-source="0"> <td style="text-align:center" data-sort="29">29</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=amobench-zh" class="llm-fu-bench-link" data-bench-id="amobench-zh" title="View models and scores"><strong>AMO-Bench CH</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Math (competition)</td> <td class="desc">Chinese subset of AMO-Bench.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="74.9"> 74.9% </td> </tr> <tr data-bench="androidworld" data-open-source="1"> <td style="text-align:center" data-sort="30">30</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=androidworld" class="llm-fu-bench-link" data-bench-id="androidworld" title="View models and scores"><strong>AndroidWorld</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> </td> <td>Mobile agents</td> <td class="desc">Benchmark for agents operating Android apps via UI automation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-35b-a3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-35B-A3B</span> </a> </td> <td data-sort="71.1"> 71.1% </td> </tr> <tr data-bench="apex-agents" data-open-source="0"> <td style="text-align:center" data-sort="31">31</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=apex-agents" class="llm-fu-bench-link" data-bench-id="apex-agents" title="View models and scores"><strong>APEX-Agents</strong></a> <span class="score-symbol" title="18 scores" aria-label="18 scores">O</span> </td> <td>Long horizon professional tasks</td> <td class="desc">APEX benchmark evaluating agents on long-horizon professional tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="33.5"> 33.5% </td> </tr> <tr data-bench="api-bank" data-open-source="1"> <td style="text-align:center" data-sort="32">32</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=api-bank" class="llm-fu-bench-link" data-bench-id="api-bank" title="View models and scores"><strong>API-Bank</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Tool use</td> <td class="desc">API-Bank tool-use benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B</span> </a> </td> <td data-sort="92.0"> 92.0% </td> </tr> <tr data-bench="arc-agi-1" data-open-source="0"> <td style="text-align:center" data-sort="33">33</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arc-agi-1" class="llm-fu-bench-link" data-bench-id="arc-agi-1" title="View models and scores"><strong>ARC-AGI-1</strong></a> <span class="score-symbol" title="106 scores" aria-label="106 scores">O</span> <a href="https://arcprize.org/leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://arcprize.org/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>General reasoning</td> <td class="desc">ARC-AGI Phase 1 aggregate accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="89.9"> 89.9% </td> </tr> <tr data-bench="arc-agi-2" data-open-source="0"> <td style="text-align:center" data-sort="34">34</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arc-agi-2" class="llm-fu-bench-link" data-bench-id="arc-agi-2" title="View models and scores"><strong>ARC-AGI-2</strong></a> <span class="score-symbol" title="119 scores" aria-label="119 scores">O</span> <a href="https://arcprize.org/leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://arcprize.org/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>General reasoning</td> <td class="desc">ARC-AGI Phase 2 aggregate accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-deep-think"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Deep Think</span> </a> </td> <td data-sort="84.6"> 84.6% </td> </tr> <tr data-bench="arc-average" data-open-source="1"> <td style="text-align:center" data-sort="35">35</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arc-average" class="llm-fu-bench-link" data-bench-id="arc-average" title="View models and scores"><strong>ARC Average</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> <a href="https://github.com/allenai/ai2-arc" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/allenai/ai2-arc" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Science QA (average)</td> <td class="desc">Average accuracy across ARC-Easy and ARC-Challenge.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="SmolLM" data-model-version="SmolLM2" data-provider="huggingface"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=smollm2-1.7b-pretrained"> <span class="brand"> <img class="logo no-invert" alt="huggingface logo" data-company="huggingface" data-country="United States" src="https://furukama.com/logos/huggingface.svg?v=5cc8e86e" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">SmolLM2 1.7B Pretrained</span> </a> </td> <td data-sort="60.5"> 60.5% </td> </tr> <tr data-bench="arc-challenge" data-open-source="1"> <td style="text-align:center" data-sort="36">36</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arc-challenge" class="llm-fu-bench-link" data-bench-id="arc-challenge" title="View models and scores"><strong>ARC-Challenge</strong></a> <span class="score-symbol" title="211 scores" aria-label="211 scores">O</span> <a href="https://github.com/allenai/ai2-arc" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Science QA</td> <td class="desc">Hard subset of AI2 Reasoning Challenge; grade-school science.</td> <td data-sort="1"> <span class="stars" title="Usage: 4/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B</span> </a> </td> <td data-sort="96.9"> 96.9% </td> </tr> <tr data-bench="arc-challenge-de" data-open-source="1"> <td style="text-align:center" data-sort="37">37</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arc-challenge-de" class="llm-fu-bench-link" data-bench-id="arc-challenge-de" title="View models and scores"><strong>ARC-Challenge (DE)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Science QA (German)</td> <td class="desc">German translation of the ARC Challenge benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="0.7"> 0.7% </td> </tr> <tr data-bench="arc-easy" data-open-source="1"> <td style="text-align:center" data-sort="38">38</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arc-easy" class="llm-fu-bench-link" data-bench-id="arc-easy" title="View models and scores"><strong>ARC-Easy</strong></a> <span class="score-symbol" title="40 scores" aria-label="40 scores">O</span> <a href="https://github.com/allenai/ai2-arc" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Science QA</td> <td class="desc">Easier subset of AI2 Reasoning Challenge.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemma" data-model-version="Gemma 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemma-3-pt-27b"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemma 3 PT 27B</span> </a> </td> <td data-sort="89.0"> 89.0% </td> </tr> <tr data-bench="arc-easy-de" data-open-source="1"> <td style="text-align:center" data-sort="39">39</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arc-easy-de" class="llm-fu-bench-link" data-bench-id="arc-easy-de" title="View models and scores"><strong>ARC-Easy (DE)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Science QA (German)</td> <td class="desc">German translation of the ARC Easy science QA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="0.8"> 0.8% </td> </tr> <tr data-bench="arena-hard" data-open-source="0"> <td style="text-align:center" data-sort="40">40</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arena-hard" class="llm-fu-bench-link" data-bench-id="arena-hard" title="View models and scores"><strong>Arena-Hard</strong></a> <span class="score-symbol" title="36 scores" aria-label="36 scores">O</span> <a href="https://github.com/lm-sys/arena-hard-auto" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Chat ability</td> <td class="desc">Hard prompts on Chatbot Arena.</td> <td data-sort="2"> <span class="stars" title="Usage: 1/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="920">920</td> <td class="leader-cell" data-model-family="Mistral" data-model-version="Mistral 3" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mistral-medium-3"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mistral Medium 3</span> </a> </td> <td data-sort="97.1"> 97.1% </td> </tr> <tr data-bench="arena-hard-v2" data-open-source="0"> <td style="text-align:center" data-sort="41">41</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arena-hard-v2" class="llm-fu-bench-link" data-bench-id="arena-hard-v2" title="View models and scores"><strong>Arena-Hard V2</strong></a> <span class="score-symbol" title="90 scores" aria-label="90 scores">O</span> <a href="https://github.com/lm-sys/arena-hard-auto" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/lmarena/arena-hard-auto" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Chat ability</td> <td class="desc">Updated Arena-Hard v2 prompts on Chatbot Arena.</td> <td data-sort="2"> <span class="stars" title="Usage: 2/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="920">920</td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-max-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3 Max Thinking</span> </a> </td> <td data-sort="90.2"> 90.2% </td> </tr> <tr data-bench="arena-hard-v2-creative-writing" data-open-source="0"> <td style="text-align:center" data-sort="42">42</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arena-hard-v2-creative-writing" class="llm-fu-bench-link" data-bench-id="arena-hard-v2-creative-writing" title="View models and scores"><strong>Arena-Hard V2 Creative Writing</strong></a> <span class="score-symbol" title="17 scores" aria-label="17 scores">O</span> <a href="https://github.com/lm-sys/arena-hard-auto" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Creative writing</td> <td class="desc">Chatbot Arena Hard V2 creative writing win-rate subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="93.6"> 93.6% </td> </tr> <tr data-bench="arena-hard-v2-hard-prompt" data-open-source="0"> <td style="text-align:center" data-sort="43">43</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arena-hard-v2-hard-prompt" class="llm-fu-bench-link" data-bench-id="arena-hard-v2-hard-prompt" title="View models and scores"><strong>Arena-Hard V2 Hard Prompt</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> <a href="https://github.com/lm-sys/arena-hard-auto" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Chat ability</td> <td class="desc">Chatbot Arena Hard V2 benchmark using the hard prompt win-rate subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="72.6"> 72.6% </td> </tr> <tr data-bench="arkitscenes" data-open-source="1"> <td style="text-align:center" data-sort="44">44</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=arkitscenes" class="llm-fu-bench-link" data-bench-id="arkitscenes" title="View models and scores"><strong>ARKitScenes</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>3D scene understanding</td> <td class="desc">ARKitScenes benchmark for assessing 3D scene reconstruction and understanding from mixed reality captures.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-vl-72b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5-VL 72B Instruct</span> </a> </td> <td data-sort="61.5"> 61.5% </td> </tr> <tr data-bench="art-agent-red-teaming" data-open-source="0"> <td style="text-align:center" data-sort="45">45</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=art-agent-red-teaming" class="llm-fu-bench-link" data-bench-id="art-agent-red-teaming" title="View models and scores"><strong>ART Agent Red Teaming</strong></a> <span class="score-symbol" title="39 scores" aria-label="39 scores">O</span> <a href="https://arxiv.org/abs/2507.20526" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Agent robustness</td> <td class="desc">Evaluation suite for adversarial red-teaming of autonomous AI agents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="33.6"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 33.6%</span> </td> </tr> <tr data-bench="artifactsbench" data-open-source="0"> <td style="text-align:center" data-sort="46">46</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=artifactsbench" class="llm-fu-bench-link" data-bench-id="artifactsbench" title="View models and scores"><strong>ArtifactsBench</strong></a> <span class="score-symbol" title="35 scores" aria-label="35 scores">O</span> <a href="https://artifactsbenchmark.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://arxiv.org/abs/2507.04952" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/Tencent-Hunyuan/ArtifactsBenchmark" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Agentic coding</td> <td class="desc">Artifacts-focused coding and tool-use benchmark evaluating generated code artifacts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-thinking"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 Thinking</span> </a> </td> <td data-sort="73.0"> 73.0% </td> </tr> <tr data-bench="asr-ami" data-open-source="1"> <td style="text-align:center" data-sort="47">47</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-ami" class="llm-fu-bench-link" data-bench-id="asr-ami" title="View models and scores"><strong>ASR AMI</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on AMI meeting speech.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5 Omni" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-omni-3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5-Omni-3B</span> </a> </td> <td data-sort="15.1"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 15.1%</span> </td> </tr> <tr data-bench="asr-earnings22" data-open-source="1"> <td style="text-align:center" data-sort="48">48</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-earnings22" class="llm-fu-bench-link" data-bench-id="asr-earnings22" title="View models and scores"><strong>ASR Earnings22</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on Earnings22 financial calls.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Whisper" data-model-version="Whisper v3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=whisper-large-v3"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Whisper-large-V3</span> </a> </td> <td data-sort="11.3"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 11.3%</span> </td> </tr> <tr data-bench="asr-gigaspeech" data-open-source="1"> <td style="text-align:center" data-sort="49">49</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-gigaspeech" class="llm-fu-bench-link" data-bench-id="asr-gigaspeech" title="View models and scores"><strong>ASR GigaSpeech</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on GigaSpeech.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Whisper" data-model-version="Whisper v3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=whisper-large-v3"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Whisper-large-V3</span> </a> </td> <td data-sort="10.0"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 10.0%</span> </td> </tr> <tr data-bench="asr-librispeech-clean" data-open-source="1"> <td style="text-align:center" data-sort="50">50</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-librispeech-clean" class="llm-fu-bench-link" data-bench-id="asr-librispeech-clean" title="View models and scores"><strong>ASR LibriSpeech Clean</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on LibriSpeech clean split.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2.5-Audio" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2.5-audio-1.5b"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2.5-Audio-1.5B</span> </a> </td> <td data-sort="1.9"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 1.9%</span> </td> </tr> <tr data-bench="asr-librispeech-other" data-open-source="1"> <td style="text-align:center" data-sort="51">51</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-librispeech-other" class="llm-fu-bench-link" data-bench-id="asr-librispeech-other" title="View models and scores"><strong>ASR LibriSpeech Other</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on LibriSpeech other split.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Whisper" data-model-version="Whisper v3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=whisper-large-v3"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Whisper-large-V3</span> </a> </td> <td data-sort="3.9"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 3.9%</span> </td> </tr> <tr data-bench="asr-spgispeech" data-open-source="1"> <td style="text-align:center" data-sort="52">52</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-spgispeech" class="llm-fu-bench-link" data-bench-id="asr-spgispeech" title="View models and scores"><strong>ASR SPGISpeech</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on SPGISpeech.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2.5-Audio" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2.5-audio-1.5b"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2.5-Audio-1.5B</span> </a> </td> <td data-sort="2.8"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 2.8%</span> </td> </tr> <tr data-bench="asr-ted-lium" data-open-source="1"> <td style="text-align:center" data-sort="53">53</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-ted-lium" class="llm-fu-bench-link" data-bench-id="asr-ted-lium" title="View models and scores"><strong>ASR TED-LIUM</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on TED-LIUM.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2.5-Audio" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2.5-audio-1.5b"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2.5-Audio-1.5B</span> </a> </td> <td data-sort="3.5"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 3.5%</span> </td> </tr> <tr data-bench="asr-voxpopuli" data-open-source="1"> <td style="text-align:center" data-sort="54">54</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=asr-voxpopuli" class="llm-fu-bench-link" data-bench-id="asr-voxpopuli" title="View models and scores"><strong>ASR VoxPopuli</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>ASR</td> <td class="desc">Automatic speech recognition benchmark on VoxPopuli.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5 Omni" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-omni-3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5-Omni-3B</span> </a> </td> <td data-sort="5.6"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 5.6%</span> </td> </tr> <tr data-bench="astabench" data-open-source="0"> <td style="text-align:center" data-sort="55">55</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=astabench" class="llm-fu-bench-link" data-bench-id="astabench" title="View models and scores"><strong>AstaBench</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> <a href="https://huggingface.co/spaces/allenai/asta-bench-leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Agent evaluation</td> <td class="desc">Evaluates science agents across literature understanding, data analysis, planning, tool use, coding, and search.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4</span> </a> </td> <td data-sort="53.0"> 53.0% </td> </tr> <tr data-bench="attaq" data-open-source="1"> <td style="text-align:center" data-sort="56">56</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=attaq" class="llm-fu-bench-link" data-bench-id="attaq" title="View models and scores"><strong>AttaQ</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Safety / jailbreak</td> <td class="desc">Adversarial jailbreak suite measuring refusal robustness against targeted attack prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Granite" data-model-version="Granite 3.3" data-provider="ibm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=granite-3.3-8b-instruct"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Granite 3.3 8B Instruct</span> </a> </td> <td data-sort="88.5"> 88.5% </td> </tr> <tr data-bench="autocodebench" data-open-source="0"> <td style="text-align:center" data-sort="57">57</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=autocodebench" class="llm-fu-bench-link" data-bench-id="autocodebench" title="View models and scores"><strong>AutoCodeBench</strong></a> <span class="score-symbol" title="42 scores" aria-label="42 scores">O</span> <a href="https://autocodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://autocodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2508.09101" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/Tencent-Hunyuan/AutoCodeBenchmark" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Autonomous coding</td> <td class="desc">End-to-end autonomous coding benchmark with unit-test based execution across diverse repositories and tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4 (Thinking)</span> </a> </td> <td data-sort="52.4"> 52.4% </td> </tr> <tr data-bench="autocodebench-lite" data-open-source="0"> <td style="text-align:center" data-sort="58">58</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=autocodebench-lite" class="llm-fu-bench-link" data-bench-id="autocodebench-lite" title="View models and scores"><strong>AutoCodeBench-Lite</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://autocodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://autocodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2508.09101" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/Tencent-Hunyuan/AutoCodeBenchmark" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Autonomous coding</td> <td class="desc">Lite version of AutoCodeBench focusing on smaller tasks with the same end-to-end, unit-test-based evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4</span> </a> </td> <td data-sort="64.5"> 64.5% </td> </tr> <tr data-bench="autologi" data-open-source="0"> <td style="text-align:center" data-sort="59">59</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=autologi" class="llm-fu-bench-link" data-bench-id="autologi" title="View models and scores"><strong>AutoLogi</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Logical reasoning</td> <td class="desc">AutoLogi benchmark evaluating automated logical reasoning accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4</span> </a> </td> <td data-sort="89.8"> 89.8% </td> </tr> <tr data-bench="babe" data-open-source="0"> <td style="text-align:center" data-sort="60">60</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=babe" class="llm-fu-bench-link" data-bench-id="babe" title="View models and scores"><strong>BABE</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>STEM reasoning</td> <td class="desc">STEM reasoning benchmark evaluating broad applied and basic engineering knowledge.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="58.1"> 58.1% </td> </tr> <tr data-bench="babyvision" data-open-source="1"> <td style="text-align:center" data-sort="61">61</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=babyvision" class="llm-fu-bench-link" data-bench-id="babyvision" title="View models and scores"><strong>BabyVision</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Visual reasoning</td> <td class="desc">Visual reasoning benchmark testing basic visual perception and understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="52.3"> 52.3% </td> </tr> <tr data-bench="balrog" data-open-source="0"> <td style="text-align:center" data-sort="62">62</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=balrog" class="llm-fu-bench-link" data-bench-id="balrog" title="View models and scores"><strong>BALROG</strong></a> <span class="score-symbol" title="27 scores" aria-label="27 scores">O</span> <a href="https://balrogai.com/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2411.13543" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/balrog-ai/BALROG" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Agent robustness</td> <td class="desc">Benchmark for assessing LLM agents under adversarial and out-of-distribution tool-use scenarios.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4</span> </a> </td> <td data-sort="43.6"> 43.6% </td> </tr> <tr data-bench="bbh" data-open-source="1"> <td style="text-align:center" data-sort="63">63</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bbh" class="llm-fu-bench-link" data-bench-id="bbh" title="View models and scores"><strong>BBH</strong></a> <span class="score-symbol" title="219 scores" aria-label="219 scores">O</span> <a href="https://github.com/suzgunmirac/BIG-bench-Hard" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2210.09261" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Multi-task reasoning</td> <td class="desc">Hard subset of BIG-bench with diverse reasoning tasks.</td> <td data-sort="3"> <span class="stars" title="Usage: 6/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="510">510</td> <td class="leader-cell" data-model-family="ERNIE" data-model-version="ERNIE 4.5" data-provider="baidu"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ernie-4.5"> <span class="brand"> <img class="logo" alt="baidu logo" data-company="baidu" data-country="China" src="https://furukama.com/logos/baidu.svg?v=c388ef3a" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">ERNIE 4.5 424B A47B</span> </a> </td> <td data-sort="94.3"> 94.3% </td> </tr> <tr data-bench="bbh-zh" data-open-source="1"> <td style="text-align:center" data-sort="64">64</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bbh-zh" class="llm-fu-bench-link" data-bench-id="bbh-zh" title="View models and scores"><strong>BBH-ZH</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Multi-task reasoning (Chinese)</td> <td class="desc">Chinese translation of BIG-Bench Hard reasoning tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LLaDA" data-model-version="LLaDA 2.0" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llada2.0-flash"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">LLaDA2.0 Flash</span> </a> </td> <td data-sort="87.5"> 87.5% </td> </tr> <tr data-bench="bbq" data-open-source="1"> <td style="text-align:center" data-sort="65">65</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bbq" class="llm-fu-bench-link" data-bench-id="bbq" title="View models and scores"><strong>BBQ</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> <a href="https://huggingface.co/datasets/heegyu/bbq" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2110.08193" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/jinjh0123/BBG" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Bias evaluation</td> <td class="desc">Bias Benchmark for Question Answering evaluating social biases across contexts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Mixtral" data-model-version="Mixtral" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mixtral-8x7b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mixtral 8x 7B</span> </a> </td> <td data-sort="56.0"> 56.0% </td> </tr> <tr data-bench="beavertails" data-open-source="1"> <td style="text-align:center" data-sort="66">66</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=beavertails" class="llm-fu-bench-link" data-bench-id="beavertails" title="View models and scores"><strong>BeaverTails</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Safety / harmfulness</td> <td class="desc">Safety benchmark evaluating harmfulness in model responses.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Thinking</span> </a> </td> <td data-sort="76.7"> 76.7% </td> </tr> <tr data-bench="beyondaime" data-open-source="0"> <td style="text-align:center" data-sort="67">67</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=beyondaime" class="llm-fu-bench-link" data-bench-id="beyondaime" title="View models and scores"><strong>BeyondAIME</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Math (beyond AIME)</td> <td class="desc">Advanced math problems exceeding AIME difficulty.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 2.0" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed2.0-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed2.0 Pro</span> </a> </td> <td data-sort="86.5"> 86.5% </td> </tr> <tr data-bench="bfcl" data-open-source="1"> <td style="text-align:center" data-sort="68">68</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bfcl" class="llm-fu-bench-link" data-bench-id="bfcl" title="View models and scores"><strong>BFCL</strong></a> <span class="score-symbol" title="17 scores" aria-label="17 scores">O</span> </td> <td>Code reasoning</td> <td class="desc">Benchmark for functional code correctness and logic.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-4b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-4B</span> </a> </td> <td data-sort="95.0"> 95.0% </td> </tr> <tr data-bench="bfcl-live-v2" data-open-source="0"> <td style="text-align:center" data-sort="69">69</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bfcl-live-v2" class="llm-fu-bench-link" data-bench-id="bfcl-live-v2" title="View models and scores"><strong>BFCL Live v2</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Finance QA</td> <td class="desc">Financial compliance and literacy questions from the BFCL Live v2 benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o1" data-model-version="o1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o1-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o1 Mini</span> </a> </td> <td data-sort="81.0"> 81.0% </td> </tr> <tr data-bench="bfcl-v2" data-open-source="1"> <td style="text-align:center" data-sort="70">70</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bfcl-v2" class="llm-fu-bench-link" data-bench-id="bfcl-v2" title="View models and scores"><strong>BFCL v2</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Code reasoning</td> <td class="desc">Second release of the BFCL benchmark focusing on functional code correctness and logic.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MobileLLM" data-model-version="MobileLLM" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mobilellm-p1"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MobileLLM P1</span> </a> </td> <td data-sort="29.4"> 29.4% </td> </tr> <tr data-bench="bfcl-v3" data-open-source="1"> <td style="text-align:center" data-sort="71">71</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bfcl-v3" class="llm-fu-bench-link" data-bench-id="bfcl-v3" title="View models and scores"><strong>BFCL v3</strong></a> <span class="score-symbol" title="87 scores" aria-label="87 scores">O</span> </td> <td>Code reasoning</td> <td class="desc">Benchmark for functional code correctness and logic (v3).</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.5" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.5"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM 4.5</span> </a> </td> <td data-sort="77.8"> 77.8% </td> </tr> <tr data-bench="bfcl-v3-live" data-open-source="1"> <td style="text-align:center" data-sort="72">72</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bfcl-v3-live" class="llm-fu-bench-link" data-bench-id="bfcl-v3-live" title="View models and scores"><strong>BFCL v3 (Live)</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Tool calling</td> <td class="desc">BFCL v3 Live subset for real-time tool calling evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Thinking-2507</span> </a> </td> <td data-sort="82.9"> 82.9% </td> </tr> <tr data-bench="bfcl-v3-multi-turn" data-open-source="1"> <td style="text-align:center" data-sort="73">73</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bfcl-v3-multi-turn" class="llm-fu-bench-link" data-bench-id="bfcl-v3-multi-turn" title="View models and scores"><strong>BFCL v3 (Multi-Turn)</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Tool calling</td> <td class="desc">BFCL v3 Multi-Turn subset for multi-turn tool calling evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiniMax" data-model-version="MiniMax M2.5" data-provider="minimax"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minimax-m2.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiniMax M2.5</span> </a> </td> <td data-sort="76.8"> 76.8% </td> </tr> <tr data-bench="bfcl-v4" data-open-source="0"> <td style="text-align:center" data-sort="74">74</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bfcl-v4" class="llm-fu-bench-link" data-bench-id="bfcl-v4" title="View models and scores"><strong>BFCL v4</strong></a> <span class="score-symbol" title="35 scores" aria-label="35 scores">O</span> </td> <td>Code reasoning</td> <td class="desc">BFCL v4 benchmark for functional code correctness and logic.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="77.5"> 77.5% </td> </tr> <tr data-bench="big-bench" data-open-source="1"> <td style="text-align:center" data-sort="75">75</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=big-bench" class="llm-fu-bench-link" data-bench-id="big-bench" title="View models and scores"><strong>BIG-Bench</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://github.com/google/BIG-bench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/google/BIG-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multi-task reasoning</td> <td class="desc">BIG-bench overall performance (original).</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="3110">3110</td> <td class="leader-cell" data-model-family="Gemma" data-model-version="Gemma 2" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemma-2-7b"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemma 2 7B</span> </a> </td> <td data-sort="55.1"> 55.1% </td> </tr> <tr data-bench="big-bench-extra-hard" data-open-source="1"> <td style="text-align:center" data-sort="76">76</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=big-bench-extra-hard" class="llm-fu-bench-link" data-bench-id="big-bench-extra-hard" title="View models and scores"><strong>BIG-Bench Extra Hard</strong></a> <span class="score-symbol" title="24 scores" aria-label="24 scores">O</span> </td> <td>Multi-task reasoning</td> <td class="desc">Extra hard subset of BIG-bench tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ling" data-model-version="Ling 2.5" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ling-2.5-1t"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ling 2.5 1T</span> </a> </td> <td data-sort="52.0"> 52.0% </td> </tr> <tr data-bench="bigcodebench" data-open-source="1"> <td style="text-align:center" data-sort="77">77</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bigcodebench" class="llm-fu-bench-link" data-bench-id="bigcodebench" title="View models and scores"><strong>BigCodeBench</strong></a> <span class="score-symbol" title="167 scores" aria-label="167 scores">O</span> <a href="https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://huggingface.co/datasets/bigcode/bigcodebench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2409.00001" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/bigcode-project/BigCodeBench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Code Generation</td> <td class="desc">BigCodeBench evaluates large language models on practical code generation tasks with unit-test verification.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiMo" data-model-version="MiMo V2" data-provider="xiaomi"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mimo-v2-flash-base"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiMo V2 Flash Base</span> </a> </td> <td data-sort="70.1"> 70.1% </td> </tr> <tr data-bench="bigcodebench-hard" data-open-source="0"> <td style="text-align:center" data-sort="78">78</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bigcodebench-hard" class="llm-fu-bench-link" data-bench-id="bigcodebench-hard" title="View models and scores"><strong>BigCodeBench Hard</strong></a> <span class="score-symbol" title="181 scores" aria-label="181 scores">O</span> <a href="https://bigcode-project.github.io/bigcodebench/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://bigcode-project.github.io/bigcodebench/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/bigcode-project/bigcodebench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Code generation (hard)</td> <td class="desc">Harder variant of BigCodeBench testing complex programming and library tasks with function-level code generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 3.7" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-3.7-sonnet-20250219"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude 3.7 Sonnet (2025-02-19)</span> </a> </td> <td data-sort="35.8"> 35.8% </td> </tr> <tr data-bench="biobench" data-open-source="0"> <td style="text-align:center" data-sort="79">79</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=biobench" class="llm-fu-bench-link" data-bench-id="biobench" title="View models and scores"><strong>BIOBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Biology reasoning</td> <td class="desc">Biology knowledge and reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="51.9"> 51.9% </td> </tr> <tr data-bench="biology-instruction" data-open-source="1"> <td style="text-align:center" data-sort="80">80</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=biology-instruction" class="llm-fu-bench-link" data-bench-id="biology-instruction" title="View models and scores"><strong>Biology-Instruction</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Biology multi-omics</td> <td class="desc">Multi-omics sequence reasoning benchmark for biological data understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="Intern-S1" data-provider="internlm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=intern-s1-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Intern-S1-Pro</span> </a> </td> <td data-sort="52.5"> 52.5% </td> </tr> <tr data-bench="biolp-bench" data-open-source="0"> <td style="text-align:center" data-sort="81">81</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=biolp-bench" class="llm-fu-bench-link" data-bench-id="biolp-bench" title="View models and scores"><strong>BioLP-Bench</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/baceolus/BioLP-bench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/baceolus/BioLP-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Biomedical NLP</td> <td class="desc">Comprehensive biomedical language processing benchmark evaluating LLMs across tasks like NER, relation extraction, and QA.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4</span> </a> </td> <td data-sort="47.0"> 47.0% </td> </tr> <tr data-bench="bird-sql" data-open-source="0"> <td style="text-align:center" data-sort="82">82</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bird-sql" class="llm-fu-bench-link" data-bench-id="bird-sql" title="View models and scores"><strong>Bird-SQL</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> </td> <td>Text-to-SQL</td> <td class="desc">Natural language to SQL generation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.0" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.0-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.0 Pro</span> </a> </td> <td data-sort="59.3"> 59.3% </td> </tr> <tr data-bench="blink" data-open-source="0"> <td style="text-align:center" data-sort="83">83</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=blink" class="llm-fu-bench-link" data-bench-id="blink" title="View models and scores"><strong>BLINK</strong></a> <span class="score-symbol" title="72 scores" aria-label="72 scores">O</span> </td> <td>Multimodal grounding</td> <td class="desc">Evaluates visual-language grounding and reference resolution to reduce hallucinations.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="87.4"> 87.4% </td> </tr> <tr data-bench="bob-hvr" data-open-source="1"> <td style="text-align:center" data-sort="84">84</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bob-hvr" class="llm-fu-bench-link" data-bench-id="bob-hvr" title="View models and scores"><strong>BoB-HVR</strong></a> <span class="score-symbol" title="32 scores" aria-label="32 scores">O</span> </td> <td>Composite capability index</td> <td class="desc">Hard, Versatile, and Relevant composite score across eight capability buckets.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3-70b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3 70B</span> </a> </td> <td data-sort="9.0"> 9.0% </td> </tr> <tr data-bench="bold" data-open-source="1"> <td style="text-align:center" data-sort="85">85</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bold" class="llm-fu-bench-link" data-bench-id="bold" title="View models and scores"><strong>BOLD</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://huggingface.co/datasets/AmazonScience/bold" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2101.11718" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/amazon-science/bold" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Bias evaluation</td> <td class="desc">Bias in Open-ended Language Dataset probing demographic biases in text generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Mixtral" data-model-version="Mixtral" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mixtral-8x7b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mixtral 8x 7B</span> </a> </td> <td data-sort="0.1"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 0.1%</span> </td> </tr> <tr data-bench="boolq" data-open-source="1"> <td style="text-align:center" data-sort="86">86</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=boolq" class="llm-fu-bench-link" data-bench-id="boolq" title="View models and scores"><strong>BoolQ</strong></a> <span class="score-symbol" title="40 scores" aria-label="40 scores">O</span> <a href="https://llm-evals.formula-labs.com/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/google-research-datasets/boolean-questions" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/pdf/1905.10044" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/google-research-datasets/boolean-questions" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Reading comprehension</td> <td class="desc">Yes/no QA from naturally occurring questions.</td> <td data-sort="2"> <span class="stars" title="Usage: 3/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="171">171</td> <td class="leader-cell" data-model-family="Marin" data-model-version="Marin 32B" data-provider="marin"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=marin-32b-mantis"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Marin-32B-Mantis</span> </a> </td> <td data-sort="89.4"> 89.4% </td> </tr> <tr data-bench="borda-count-multilingual" data-open-source="1"> <td style="text-align:center" data-sort="87">87</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=borda-count-multilingual" class="llm-fu-bench-link" data-bench-id="borda-count-multilingual" title="View models and scores"><strong>Borda Count (Multilingual)</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Aggregate ranking</td> <td class="desc">Borda count aggregate ranking across multilingual benchmarks; lower is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-32b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-32B</span> </a> </td> <td data-sort="2.9"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 2.9%</span> </td> </tr> <tr data-bench="bridgebench" data-open-source="0"> <td style="text-align:center" data-sort="88">88</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=bridgebench" class="llm-fu-bench-link" data-bench-id="bridgebench" title="View models and scores"><strong>BridgeBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Reasoning</td> <td class="desc">BridgeBench evaluation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="60.1"> 60.1% </td> </tr> <tr data-bench="browsecomp" data-open-source="0"> <td style="text-align:center" data-sort="89">89</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=browsecomp" class="llm-fu-bench-link" data-bench-id="browsecomp" title="View models and scores"><strong>BrowseComp</strong></a> <span class="score-symbol" title="82 scores" aria-label="82 scores">O</span> <a href="https://openai.com/index/browsecomp/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Web browsing</td> <td class="desc">Web browsing comprehension and competence benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="85.9"> 85.9% </td> </tr> <tr data-bench="browsecomp-with-content-manager" data-open-source="0"> <td style="text-align:center" data-sort="90">90</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=browsecomp-with-content-manager" class="llm-fu-bench-link" data-bench-id="browsecomp-with-content-manager" title="View models and scores"><strong>BrowseComp (With Content Manager)</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> <a href="https://openai.com/index/browsecomp/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Web browsing</td> <td class="desc">BrowseComp benchmark evaluated with content manager assistance.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="84.0"> 84.0% </td> </tr> <tr data-bench="browsecomp-zh" data-open-source="0"> <td style="text-align:center" data-sort="91">91</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=browsecomp-zh" class="llm-fu-bench-link" data-bench-id="browsecomp-zh" title="View models and scores"><strong>BrowseComp_zh</strong></a> <span class="score-symbol" title="52 scores" aria-label="52 scores">O</span> </td> <td>Web browsing (Chinese)</td> <td class="desc">Chinese variant of the BrowseComp web browsing benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="81.3"> 81.3% </td> </tr> <tr data-bench="brumo25" data-open-source="1"> <td style="text-align:center" data-sort="92">92</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=brumo25" class="llm-fu-bench-link" data-bench-id="brumo25" title="View models and scores"><strong>BRuMo25</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math competition</td> <td class="desc">BruMo 2025 olympiad-style mathematics benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="QuestA Nemotron" data-provider="amazon"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=questa-nemotron-1.5b"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">QuestA Nemotron 1.5B</span> </a> </td> <td data-sort="69.5"> 69.5% </td> </tr> <tr data-bench="buzzbench" data-open-source="0"> <td style="text-align:center" data-sort="93">93</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=buzzbench" class="llm-fu-bench-link" data-bench-id="buzzbench" title="View models and scores"><strong>BuzzBench</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://eqbench.com/buzzbench.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://arxiv.org/abs/2312.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/EQ-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Humor analysis</td> <td class="desc">A humour analysis benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="71.1"> 71.1% </td> </tr> <tr data-bench="c-eval" data-open-source="1"> <td style="text-align:center" data-sort="94">94</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=c-eval" class="llm-fu-bench-link" data-bench-id="c-eval" title="View models and scores"><strong>C-Eval</strong></a> <span class="score-symbol" title="52 scores" aria-label="52 scores">O</span> <a href="https://github.com/SJTU-LIT/ceval" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Chinese exams</td> <td class="desc">Chinese college-level exam benchmark.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1768">1768</td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5</span> </a> </td> <td data-sort="94.0"> 94.0% </td> </tr> <tr data-bench="c3-bench" data-open-source="1"> <td style="text-align:center" data-sort="95">95</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=c3-bench" class="llm-fu-bench-link" data-bench-id="c3-bench" title="View models and scores"><strong>C3-Bench</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/Tencent-Hunyuan/C3-Benchmark" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Reasoning (Chinese)</td> <td class="desc">Comprehensive Chinese reasoning capability benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="35">35</td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.5" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.5-base"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.5 Base</span> </a> </td> <td data-sort="83.1"> 83.1% </td> </tr> <tr data-bench="caselaw-v2" data-open-source="0"> <td style="text-align:center" data-sort="96">96</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=caselaw-v2" class="llm-fu-bench-link" data-bench-id="caselaw-v2" title="View models and scores"><strong>CaseLaw v2</strong></a> <span class="score-symbol" title="30 scores" aria-label="30 scores">O</span> <a href="https://www.vals.ai/benchmarks/case_law_v2-09-20-2025" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://www.vals.ai/benchmarks/case_law_v2-09-20-2025" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Legal reasoning</td> <td class="desc">U.S. case law benchmark evaluating legal reasoning and judgment over court opinions.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4.1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4.1"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4.1</span> </a> </td> <td data-sort="78.1"> 78.1% </td> </tr> <tr data-bench="cc-ocr" data-open-source="1"> <td style="text-align:center" data-sort="97">97</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cc-ocr" class="llm-fu-bench-link" data-bench-id="cc-ocr" title="View models and scores"><strong>CC-OCR</strong></a> <span class="score-symbol" title="26 scores" aria-label="26 scores">O</span> </td> <td>OCR (cross-lingual)</td> <td class="desc">Cross-lingual OCR benchmark evaluating character recognition across mixed-language documents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="82.0"> 82.0% </td> </tr> <tr data-bench="cfeval" data-open-source="1"> <td style="text-align:center" data-sort="98">98</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cfeval" class="llm-fu-bench-link" data-bench-id="cfeval" title="View models and scores"><strong>CFEval</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Coding ELO / contest eval</td> <td class="desc">Contest-style coding evaluation with ELO-like scoring.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-235b-a22b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-235B-A22B-Thinking-2507</span> </a> </td> <td data-sort="2134"> 2134 </td> </tr> <tr data-bench="cgbench" data-open-source="0"> <td style="text-align:center" data-sort="99">99</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cgbench" class="llm-fu-bench-link" data-bench-id="cgbench" title="View models and scores"><strong>CGBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Long video QA</td> <td class="desc">Cartoon/CG long video question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="64.6"> 64.6% </td> </tr> <tr data-bench="charades-sta" data-open-source="0"> <td style="text-align:center" data-sort="100">100</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=charades-sta" class="llm-fu-bench-link" data-bench-id="charades-sta" title="View models and scores"><strong>Charades-STA</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> <a href="https://huggingface.co/datasets/VLM2Vec/Charades-STA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Video grounding</td> <td class="desc">Charades-STA temporal grounding (mIoU).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.5" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.5-vl-thinking"> <span class="brand"> <img class="logo no-invert" alt="bytedance logo" data-company="bytedance" data-country="China" src="https://furukama.com/logos/bytedance.svg?v=4e7ed8af" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Seed1.5-VL-Thinking</span> </a> </td> <td data-sort="64.0"> 64.0% </td> </tr> <tr data-bench="chartmuseum" data-open-source="0"> <td style="text-align:center" data-sort="101">101</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=chartmuseum" class="llm-fu-bench-link" data-bench-id="chartmuseum" title="View models and scores"><strong>ChartMuseum</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://chartmuseum-leaderboard.github.io/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://chartmuseum-leaderboard.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2505.13444" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/Liyan06/ChartMuseum" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Chart understanding</td> <td class="desc">Large-scale curated collection of charts for evaluating parsing, grounding, and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 mini</span> </a> </td> <td data-sort="63.3"> 63.3% </td> </tr> <tr data-bench="chartqa" data-open-source="1"> <td style="text-align:center" data-sort="102">102</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=chartqa" class="llm-fu-bench-link" data-bench-id="chartqa" title="View models and scores"><strong>ChartQA</strong></a> <span class="score-symbol" title="110 scores" aria-label="110 scores">O</span> <a href="https://github.com/vis-nlp/ChartQA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Chart understanding (VQA)</td> <td class="desc">Visual question answering over charts and plots.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Keye-VL" data-model-version="Keye-VL 1.5" data-provider="kuaishou"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=keye-vl-1.5-8b"> <span class="brand"> <img class="logo no-invert" alt="kuaishou logo" data-company="kuaishou" data-country="China" src="https://furukama.com/logos/kuaishou.svg?v=a2671fc0" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Keye-VL-1.5-8B</span> </a> </td> <td data-sort="94.1"> 94.1% </td> </tr> <tr data-bench="chartqa-pro" data-open-source="0"> <td style="text-align:center" data-sort="103">103</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=chartqa-pro" class="llm-fu-bench-link" data-bench-id="chartqa-pro" title="View models and scores"><strong>ChartQA-Pro</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Chart understanding (VQA)</td> <td class="desc">Professional-grade chart question answering with diverse chart types and complex reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="69.5"> 69.5% </td> </tr> <tr data-bench="charxiv-dq" data-open-source="0"> <td style="text-align:center" data-sort="104">104</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=charxiv-dq" class="llm-fu-bench-link" data-bench-id="charxiv-dq" title="View models and scores"><strong>CharXiv (DQ)</strong></a> <span class="score-symbol" title="30 scores" aria-label="30 scores">O</span> <a href="https://charxiv.github.io/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://charxiv.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Chart description (PDF)</td> <td class="desc">Scientific chart/table descriptive questions from arXiv PDFs.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o3" data-model-version="o3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o3-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o3-high</span> </a> </td> <td data-sort="95.0"> 95.0% </td> </tr> <tr data-bench="charxiv-rq" data-open-source="0"> <td style="text-align:center" data-sort="105">105</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=charxiv-rq" class="llm-fu-bench-link" data-bench-id="charxiv-rq" title="View models and scores"><strong>CharXiv (RQ)</strong></a> <span class="score-symbol" title="71 scores" aria-label="71 scores">O</span> <a href="https://charxiv.github.io/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://charxiv.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Chart reasoning (PDF)</td> <td class="desc">Scientific chart/table reasoning questions from arXiv PDFs.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-thinking"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Thinking</span> </a> </td> <td data-sort="82.1"> 82.1% </td> </tr> <tr data-bench="chinese-simpleqa" data-open-source="1"> <td style="text-align:center" data-sort="106">106</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=chinese-simpleqa" class="llm-fu-bench-link" data-bench-id="chinese-simpleqa" title="View models and scores"><strong>Chinese SimpleQA</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>QA (Chinese)</td> <td class="desc">Chinese variant of the SimpleQA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-base"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2 Base</span> </a> </td> <td data-sort="77.6"> 77.6% </td> </tr> <tr data-bench="cl-bench" data-open-source="0"> <td style="text-align:center" data-sort="107">107</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cl-bench" class="llm-fu-bench-link" data-bench-id="cl-bench" title="View models and scores"><strong>CL-Bench</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Comprehensive long-context benchmark evaluating reasoning over extended contexts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-mini-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 Mini High</span> </a> </td> <td data-sort="25.2"> 25.2% </td> </tr> <tr data-bench="click" data-open-source="1"> <td style="text-align:center" data-sort="108">108</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=click" class="llm-fu-bench-link" data-bench-id="click" title="View models and scores"><strong>CLIcK</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Korean instruction following</td> <td class="desc">Korean long-form instruction-following benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.2-thinking"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.2-Thinking</span> </a> </td> <td data-sort="86.3"> 86.3% </td> </tr> <tr data-bench="cloningscenarios" data-open-source="0"> <td style="text-align:center" data-sort="109">109</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cloningscenarios" class="llm-fu-bench-link" data-bench-id="cloningscenarios" title="View models and scores"><strong>CloningScenarios</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Biosecurity refusal</td> <td class="desc">Safety benchmark that red-teams models with cloning-related misuse scenarios to measure compliance and refusal rates.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4</span> </a> </td> <td data-sort="45.0"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 45.0%</span> </td> </tr> <tr data-bench="cluewsc" data-open-source="1"> <td style="text-align:center" data-sort="110">110</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cluewsc" class="llm-fu-bench-link" data-bench-id="cluewsc" title="View models and scores"><strong>CLUEWSC</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://www.cluebenchmarks.com/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Coreference reasoning (Chinese)</td> <td class="desc">Chinese Winograd Schema-style coreference benchmark from CLUE.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek R1" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-r1"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek R1</span> </a> </td> <td data-sort="92.8"> 92.8% </td> </tr> <tr data-bench="cmath" data-open-source="1"> <td style="text-align:center" data-sort="111">111</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cmath" class="llm-fu-bench-link" data-bench-id="cmath" title="View models and scores"><strong>CMath</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Math (Chinese)</td> <td class="desc">Chinese mathematics benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LLaDA" data-model-version="LLaDA 2.0" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llada2.0-flash"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">LLaDA2.0 Flash</span> </a> </td> <td data-sort="96.9"> 96.9% </td> </tr> <tr data-bench="cmmlu" data-open-source="0"> <td style="text-align:center" data-sort="112">112</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cmmlu" class="llm-fu-bench-link" data-bench-id="cmmlu" title="View models and scores"><strong>CMMLU</strong></a> <span class="score-symbol" title="34 scores" aria-label="34 scores">O</span> <a href="https://github.com/haonan-li/CMMLU" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Chinese multi-domain</td> <td class="desc">Chinese counterpart to MMLU.</td> <td data-sort="2"> <span class="stars" title="Usage: 3/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="781">781</td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen-2.5-max"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5 Max</span> </a> </td> <td data-sort="91.9"> 91.9% </td> </tr> <tr data-bench="cnmo-2024" data-open-source="1"> <td style="text-align:center" data-sort="113">113</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cnmo-2024" class="llm-fu-bench-link" data-bench-id="cnmo-2024" title="View models and scores"><strong>CNMO 2024</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Math (competition)</td> <td class="desc">China National Mathematical Olympiad 2024 evaluation set.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="openPangu-R" data-model-version="openPangu-R" data-provider="freedomintelligence"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=openpangu-r-72b-2512-slow-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">openPangu-R-72B-2512 Slow Thinking</span> </a> </td> <td data-sort="82.8"> 82.8% </td> </tr> <tr data-bench="codeforces" data-open-source="0"> <td style="text-align:center" data-sort="114">114</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=codeforces" class="llm-fu-bench-link" data-bench-id="codeforces" title="View models and scores"><strong>Codeforces</strong></a> <span class="score-symbol" title="76 scores" aria-label="76 scores">O</span> <a href="https://llmdb.com/benchmarks/codeforces" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://codeforces.com/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Competitive programming</td> <td class="desc">Competitive programming performance on Codeforces problems (ELO).</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-deep-think"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Deep Think</span> </a> </td> <td data-sort="3455"> 3455 </td> </tr> <tr data-bench="codeif-bench" data-open-source="1"> <td style="text-align:center" data-sort="115">115</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=codeif-bench" class="llm-fu-bench-link" data-bench-id="codeif-bench" title="View models and scores"><strong>CodeIF-Bench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Code instruction following</td> <td class="desc">Code-focused instruction following benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-8b-non-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-8B Non-Thinking</span> </a> </td> <td data-sort="50.0"> 50.0% </td> </tr> <tr data-bench="collie" data-open-source="0"> <td style="text-align:center" data-sort="116">116</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=collie" class="llm-fu-bench-link" data-bench-id="collie" title="View models and scores"><strong>COLLIE</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> <a href="https://collie-benchmark.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Instruction following</td> <td class="desc">Comprehensive instruction-following evaluation suite.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="55">55</td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="99.0"> 99.0% </td> </tr> <tr data-bench="collie-hard" data-open-source="0"> <td style="text-align:center" data-sort="117">117</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=collie-hard" class="llm-fu-bench-link" data-bench-id="collie-hard" title="View models and scores"><strong>Collie-Hard</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Instruction following</td> <td class="desc">Hard subset of Collie instruction-following tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="99.0"> 99.0% </td> </tr> <tr data-bench="commonsenseqa" data-open-source="1"> <td style="text-align:center" data-sort="118">118</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=commonsenseqa" class="llm-fu-bench-link" data-bench-id="commonsenseqa" title="View models and scores"><strong>CommonsenseQA</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> <a href="https://www.tau-nlp.org/csqa" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Commonsense QA</td> <td class="desc">Multiple-choice QA requiring commonsense knowledge.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-32b-base"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5 32B Base</span> </a> </td> <td data-sort="88.5"> 88.5% </td> </tr> <tr data-bench="complex-workflow" data-open-source="0"> <td style="text-align:center" data-sort="119">119</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=complex-workflow" class="llm-fu-bench-link" data-bench-id="complex-workflow" title="View models and scores"><strong>Complex Workflow</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Complex workflows</td> <td class="desc">Complex workflow benchmark for economically valuable tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="58.2"> 58.2% </td> </tr> <tr data-bench="copa" data-open-source="1"> <td style="text-align:center" data-sort="120">120</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=copa" class="llm-fu-bench-link" data-bench-id="copa" title="View models and scores"><strong>COPA</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://people.ict.usc.edu/~gordon/copa.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Causal reasoning</td> <td class="desc">Choice of Plausible Alternatives.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Marin" data-model-version="Marin 32B" data-provider="marin"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=marin-32b-bison"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Marin-32B-Bison</span> </a> </td> <td data-sort="94.0"> 94.0% </td> </tr> <tr data-bench="core" data-open-source="1"> <td style="text-align:center" data-sort="121">121</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=core" class="llm-fu-bench-link" data-bench-id="core" title="View models and scores"><strong>CORE</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Ontological reasoning</td> <td class="desc">Comprehensive Ontological Relation Evaluation for Large Language Models.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nanbeige" data-model-version="Nanbeige4.1" data-provider="boss"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nanbeige4.1-3b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Nanbeige4.1-3B</span> </a> </td> <td data-sort="53.5"> 53.5% </td> </tr> <tr data-bench="corpusqa" data-open-source="0"> <td style="text-align:center" data-sort="122">122</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=corpusqa" class="llm-fu-bench-link" data-bench-id="corpusqa" title="View models and scores"><strong>CorpusQA</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Long-context QA</td> <td class="desc">Question answering over large text corpora.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="81.6"> 81.6% </td> </tr> <tr data-bench="countbench" data-open-source="1"> <td style="text-align:center" data-sort="123">123</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=countbench" class="llm-fu-bench-link" data-bench-id="countbench" title="View models and scores"><strong>CountBench</strong></a> <span class="score-symbol" title="66 scores" aria-label="66 scores">O</span> <a href="https://huggingface.co/datasets/nielsr/countbench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2302.12066" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Visual counting</td> <td class="desc">Object counting and numeracy benchmark for visual-language models across varied scenes.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-27b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-27B</span> </a> </td> <td data-sort="97.8"> 97.8% </td> </tr> <tr data-bench="countbenchqa" data-open-source="1"> <td style="text-align:center" data-sort="124">124</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=countbenchqa" class="llm-fu-bench-link" data-bench-id="countbenchqa" title="View models and scores"><strong>CountBenchQA</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Visual counting QA</td> <td class="desc">Visual question answering benchmark focused on counting objects across varied scenes.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Moondream" data-model-version="Moondream 3" data-provider="moondream"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=moondream-9b-a2b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Moondream-9B-A2B</span> </a> </td> <td data-sort="93.2"> 93.2% </td> </tr> <tr data-bench="countdown" data-open-source="1"> <td style="text-align:center" data-sort="125">125</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=countdown" class="llm-fu-bench-link" data-bench-id="countdown" title="View models and scores"><strong>Countdown</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Planning and reasoning</td> <td class="desc">Countdown-style reasoning and planning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="K2" data-model-version="K2 V2" data-provider="mbzuai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k2-v2"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K2-V2</span> </a> </td> <td data-sort="75.6"> 75.6% </td> </tr> <tr data-bench="countix" data-open-source="0"> <td style="text-align:center" data-sort="126">126</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=countix" class="llm-fu-bench-link" data-bench-id="countix" title="View models and scores"><strong>Countix</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Video counting</td> <td class="desc">Video-based counting benchmark for multiple objects.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="31.0"> 31.0% </td> </tr> <tr data-bench="crag" data-open-source="0"> <td style="text-align:center" data-sort="127">127</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=crag" class="llm-fu-bench-link" data-bench-id="crag" title="View models and scores"><strong>CRAG</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Retrieval QA</td> <td class="desc">Complex Retrieval-Augmented Generation benchmark for grounded question answering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Jamba" data-model-version="Jamba Mini" data-provider="ai21"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=jamba-mini-1.6"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Jamba Mini 1.6</span> </a> </td> <td data-sort="76.2"> 76.2% </td> </tr> <tr data-bench="creative-story-writing-v3" data-open-source="1"> <td style="text-align:center" data-sort="128">128</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=creative-story-writing-v3" class="llm-fu-bench-link" data-bench-id="creative-story-writing-v3" title="View models and scores"><strong>Creative Story‑Writing Benchmark V3</strong></a> <span class="score-symbol" title="17 scores" aria-label="17 scores">O</span> <a href="https://github.com/lechmazur/writing" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/lechmazur/writing" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Creative writing</td> <td class="desc">Story writing benchmark evaluating creativity, coherence, and style (v3).</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="291">291</td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-instruct-0905"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2-Instruct-0905</span> </a> </td> <td data-sort="8.7"> 8.7% </td> </tr> <tr data-bench="creative-writing-longform" data-open-source="0"> <td style="text-align:center" data-sort="129">129</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=creative-writing-longform" class="llm-fu-bench-link" data-bench-id="creative-writing-longform" title="View models and scores"><strong>Longform Creative Writing</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> <a href="https://eqbench.com/creative_writing_longform.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://eqbench.com/creative_writing_longform.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2312.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/EQ-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Creative writing</td> <td class="desc">Longform creative writing evaluation (EQ-Bench).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="20">20</td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-4.5-sonnet-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5 (Thinking)</span> </a> </td> <td data-sort="79.8"> 79.8% </td> </tr> <tr data-bench="creative-writing-v3" data-open-source="0"> <td style="text-align:center" data-sort="130">130</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=creative-writing-v3" class="llm-fu-bench-link" data-bench-id="creative-writing-v3" title="View models and scores"><strong>Creative Writing v3</strong></a> <span class="score-symbol" title="64 scores" aria-label="64 scores">O</span> <a href="https://eqbench.com/creative_writing.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://eqbench.com/creative_writing.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2312.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/EQ-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Creative writing</td> <td class="desc">A LLM-judged creative writing benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="54">54</td> <td class="leader-cell" data-model-family="o3" data-model-version="o3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o3"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o3</span> </a> </td> <td data-sort="1661"> 1661 </td> </tr> <tr data-bench="critpt" data-open-source="0"> <td style="text-align:center" data-sort="131">131</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=critpt" class="llm-fu-bench-link" data-bench-id="critpt" title="View models and scores"><strong>Complex Research using Integrated Thinking – Physics Test</strong></a> <span class="score-symbol" title="38 scores" aria-label="38 scores">O</span> <a href="https://critpt.com/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Reasoning</td> <td class="desc">CritPt (Complex Research using Integrated Thinking – Physics Test) benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high-code-web"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 (High, Code &amp; Web)</span> </a> </td> <td data-sort="12.6"> 12.6% </td> </tr> <tr data-bench="crux-i" data-open-source="0"> <td style="text-align:center" data-sort="132">132</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=crux-i" class="llm-fu-bench-link" data-bench-id="crux-i" title="View models and scores"><strong>CRUX-I</strong></a> <span class="score-symbol" title="74 scores" aria-label="74 scores">O</span> <a href="https://crux-eval.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Code reasoning</td> <td class="desc">Code Reasoning and Understanding eXam – Interactive.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro-preview"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro Preview</span> </a> </td> <td data-sort="98.8"> 98.8% </td> </tr> <tr data-bench="crux-o" data-open-source="1"> <td style="text-align:center" data-sort="133">133</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=crux-o" class="llm-fu-bench-link" data-bench-id="crux-o" title="View models and scores"><strong>CRUX-O</strong></a> <span class="score-symbol" title="138 scores" aria-label="138 scores">O</span> <a href="https://crux-eval.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Code reasoning</td> <td class="desc">Code Reasoning and Understanding eXam – Offline.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-loop-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Loop-Thinking</span> </a> </td> <td data-sort="99.4"> 99.4% </td> </tr> <tr data-bench="cruxeval" data-open-source="1"> <td style="text-align:center" data-sort="134">134</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cruxeval" class="llm-fu-bench-link" data-bench-id="cruxeval" title="View models and scores"><strong>CruxEval</strong></a> <span class="score-symbol" title="40 scores" aria-label="40 scores">O</span> <a href="https://github.com/crux-eval/cruxeval" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/crux-eval/cruxeval" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Code reasoning</td> <td class="desc">Mathematical coding challenge set from the CruxEval benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-instruct-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Instruct-2507</span> </a> </td> <td data-sort="86.8"> 86.8% </td> </tr> <tr data-bench="csimpleqa" data-open-source="1"> <td style="text-align:center" data-sort="135">135</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=csimpleqa" class="llm-fu-bench-link" data-bench-id="csimpleqa" title="View models and scores"><strong>CSimpleQA</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>QA</td> <td class="desc">Chinese SimpleQA benchmark variant (short factual questions).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ling" data-model-version="Ling 2.5" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ling-2.5-1t"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ling 2.5 1T</span> </a> </td> <td data-sort="79.0"> 79.0% </td> </tr> <tr data-bench="customer-support-qa" data-open-source="0"> <td style="text-align:center" data-sort="136">136</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=customer-support-qa" class="llm-fu-bench-link" data-bench-id="customer-support-qa" title="View models and scores"><strong>Customer Support Q&amp;A</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Customer support QA</td> <td class="desc">Customer support question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="69.0"> 69.0% </td> </tr> <tr data-bench="cute" data-open-source="1"> <td style="text-align:center" data-sort="137">137</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cute" class="llm-fu-bench-link" data-bench-id="cute" title="View models and scores"><strong>CUTE</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>English characters</td> <td class="desc">CUTE aggregate capability score.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Bolmo" data-model-version="Bolmo 7B" data-provider="allenai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=bolmo-7b"> <span class="brand"> <img class="logo" alt="allenai logo" data-company="allenai" data-country="United States" src="https://furukama.com/logos/allenai.svg?v=27b5cbbf" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Bolmo 7B</span> </a> </td> <td data-sort="78.6"> 78.6% </td> </tr> <tr data-bench="cv-bench" data-open-source="0"> <td style="text-align:center" data-sort="138">138</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cv-bench" class="llm-fu-bench-link" data-bench-id="cv-bench" title="View models and scores"><strong>CV-Bench</strong></a> <span class="score-symbol" title="25 scores" aria-label="25 scores">O</span> </td> <td>Computer vision QA</td> <td class="desc">Diverse CV tasks for VLMs.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="92.0"> 92.0% </td> </tr> <tr data-bench="cvtg-2k-clipscore" data-open-source="0"> <td style="text-align:center" data-sort="139">139</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cvtg-2k-clipscore" class="llm-fu-bench-link" data-bench-id="cvtg-2k-clipscore" title="View models and scores"><strong>CVTG-2K CLIPScore</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Text rendering</td> <td class="desc">CVTG-2K CLIPScore for text rendering in image generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seedream" data-model-version="Seedream" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seedream-4.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seedream 4.5</span> </a> </td> <td data-sort="0.8"> 0.8% </td> </tr> <tr data-bench="cvtg-2k-ned" data-open-source="1"> <td style="text-align:center" data-sort="140">140</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cvtg-2k-ned" class="llm-fu-bench-link" data-bench-id="cvtg-2k-ned" title="View models and scores"><strong>CVTG-2K NED</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Text rendering</td> <td class="desc">CVTG-2K normalized edit distance (NED) for text rendering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM-Image" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-image"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-Image</span> </a> </td> <td data-sort="1.0"> 1.0% </td> </tr> <tr data-bench="cvtg-2k-word-accuracy" data-open-source="1"> <td style="text-align:center" data-sort="141">141</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cvtg-2k-word-accuracy" class="llm-fu-bench-link" data-bench-id="cvtg-2k-word-accuracy" title="View models and scores"><strong>CVTG-2K Word Accuracy</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Text rendering</td> <td class="desc">CVTG-2K word accuracy for text rendering in images.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM-Image" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-image"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-Image</span> </a> </td> <td data-sort="0.9"> 0.9% </td> </tr> <tr data-bench="cybench" data-open-source="0"> <td style="text-align:center" data-sort="142">142</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cybench" class="llm-fu-bench-link" data-bench-id="cybench" title="View models and scores"><strong>CyBench</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://cybench.github.io/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://cybench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Cybersecurity CTF</td> <td class="desc">Framework with 40 professional-level CTF tasks evaluating LLMs&#39; practical cybersecurity capabilities.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o3" data-model-version="o3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o3-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o3 mini</span> </a> </td> <td data-sort="22.5"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 22.5%</span> </td> </tr> <tr data-bench="cybergym" data-open-source="0"> <td style="text-align:center" data-sort="143">143</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cybergym" class="llm-fu-bench-link" data-bench-id="cybergym" title="View models and scores"><strong>CyberGym</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Cybersecurity tasks</td> <td class="desc">Benchmark for cybersecurity-related coding and reasoning tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5 Thinking</span> </a> </td> <td data-sort="50.6"> 50.6% </td> </tr> <tr data-bench="cybersecurity-capture-the-flag-challenges" data-open-source="0"> <td style="text-align:center" data-sort="144">144</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cybersecurity-capture-the-flag-challenges" class="llm-fu-bench-link" data-bench-id="cybersecurity-capture-the-flag-challenges" title="View models and scores"><strong>Cybersecurity Capture The Flag Challenges</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Cybersecurity CTF</td> <td class="desc">Capture-the-flag challenge benchmark evaluating cybersecurity problem-solving skills.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.3 Codex" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.3-codex"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.3 Codex</span> </a> </td> <td data-sort="77.6"> 77.6% </td> </tr> <tr data-bench="cybersecurity-ctf" data-open-source="0"> <td style="text-align:center" data-sort="145">145</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=cybersecurity-ctf" class="llm-fu-bench-link" data-bench-id="cybersecurity-ctf" title="View models and scores"><strong>Cybersecurity CTF</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Cybersecurity CTF</td> <td class="desc">Cybersecurity Capture The Flag challenges benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.3 Codex" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.3-codex"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.3 Codex</span> </a> </td> <td data-sort="77.6"> 77.6% </td> </tr> <tr data-bench="da-2k" data-open-source="0"> <td style="text-align:center" data-sort="146">146</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=da-2k" class="llm-fu-bench-link" data-bench-id="da-2k" title="View models and scores"><strong>DA-2K</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Spatial reasoning</td> <td class="desc">2D/3D spatial reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.5" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.5-vl-thinking"> <span class="brand"> <img class="logo no-invert" alt="bytedance logo" data-company="bytedance" data-country="China" src="https://furukama.com/logos/bytedance.svg?v=4e7ed8af" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Seed1.5-VL-Thinking</span> </a> </td> <td data-sort="85.3"> 85.3% </td> </tr> <tr data-bench="deep-planning" data-open-source="0"> <td style="text-align:center" data-sort="147">147</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=deep-planning" class="llm-fu-bench-link" data-bench-id="deep-planning" title="View models and scores"><strong>Deep Planning</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>Planning and reasoning</td> <td class="desc">Benchmark evaluating deep planning and multi-step reasoning capabilities.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-thinking"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Thinking</span> </a> </td> <td data-sort="44.6"> 44.6% </td> </tr> <tr data-bench="deepconsult" data-open-source="0"> <td style="text-align:center" data-sort="148">148</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=deepconsult" class="llm-fu-bench-link" data-bench-id="deepconsult" title="View models and scores"><strong>DeepConsult</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Agentic writing</td> <td class="desc">Agentic consulting and writing benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="57.2"> 57.2% </td> </tr> <tr data-bench="deepmind-math" data-open-source="1"> <td style="text-align:center" data-sort="149">149</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=deepmind-math" class="llm-fu-bench-link" data-bench-id="deepmind-math" title="View models and scores"><strong>DeepMind Mathematics</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://github.com/deepmind/mathematics_dataset" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/1904.01557" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/deepmind/mathematics_dataset" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Math reasoning</td> <td class="desc">Synthetic math problem sets from DeepMind covering arithmetic, algebra, calculus, and more.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Granite" data-model-version="Granite 4.0" data-provider="ibm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=granite-4.0-h-small"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Granite-4.0-H-Small</span> </a> </td> <td data-sort="59.3"> 59.3% </td> </tr> <tr data-bench="deepresearchbench" data-open-source="0"> <td style="text-align:center" data-sort="150">150</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=deepresearchbench" class="llm-fu-bench-link" data-bench-id="deepresearchbench" title="View models and scores"><strong>DeepResearchBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Agentic research writing</td> <td class="desc">Research-oriented agentic writing and planning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="49.6"> 49.6% </td> </tr> <tr data-bench="deepsearchqa" data-open-source="1"> <td style="text-align:center" data-sort="151">151</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=deepsearchqa" class="llm-fu-bench-link" data-bench-id="deepsearchqa" title="View models and scores"><strong>DeepSearchQA</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Deep web search QA</td> <td class="desc">Multi-step web search and question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5 Thinking</span> </a> </td> <td data-sort="77.1"> 77.1% </td> </tr> <tr data-bench="der2-bench" data-open-source="0"> <td style="text-align:center" data-sort="152">152</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=der2-bench" class="llm-fu-bench-link" data-bench-id="der2-bench" title="View models and scores"><strong>DeR2 Bench</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Dense retrieval and reasoning benchmark for long-context evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="69.0"> 69.0% </td> </tr> <tr data-bench="design2code" data-open-source="1"> <td style="text-align:center" data-sort="153">153</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=design2code" class="llm-fu-bench-link" data-bench-id="design2code" title="View models and scores"><strong>Design2Code</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Coding (UI)</td> <td class="desc">Translating UI designs into code.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-235b-a22b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-235B-A22B Thinking</span> </a> </td> <td data-sort="93.4"> 93.4% </td> </tr> <tr data-bench="designarena" data-open-source="0"> <td style="text-align:center" data-sort="154">154</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=designarena" class="llm-fu-bench-link" data-bench-id="designarena" title="View models and scores"><strong>DesignArena</strong></a> <span class="score-symbol" title="77 scores" aria-label="77 scores">O</span> <a href="https://designarena.ai" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://designarena.ai" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Generative design</td> <td class="desc">Leaderboard tracking generative design systems across layout, branding, and marketing tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-4.5-sonnet-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5 (Thinking)</span> </a> </td> <td data-sort="1410"> 1410 </td> </tr> <tr data-bench="detailbench" data-open-source="1"> <td style="text-align:center" data-sort="155">155</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=detailbench" class="llm-fu-bench-link" data-bench-id="detailbench" title="View models and scores"><strong>DetailBench</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://huggingface.co/datasets/xeophon/detailbench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Spot small mistakes</td> <td class="desc">Evaluates whether LLMs can notice subtle errors and minor inconsistencies in text.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 4" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-4-maverick"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 4 Maverick</span> </a> </td> <td data-sort="8.7"> 8.7% </td> </tr> <tr data-bench="discox" data-open-source="0"> <td style="text-align:center" data-sort="156">156</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=discox" class="llm-fu-bench-link" data-bench-id="discox" title="View models and scores"><strong>DiscoX</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Agentic writing</td> <td class="desc">DiscoX benchmark for agentic writing and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 2.0" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed2.0-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed2.0 Pro</span> </a> </td> <td data-sort="82.0"> 82.0% </td> </tr> <tr data-bench="do-anything-now" data-open-source="1"> <td style="text-align:center" data-sort="157">157</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=do-anything-now" class="llm-fu-bench-link" data-bench-id="do-anything-now" title="View models and scores"><strong>Do-Anything-Now</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Safety / jailbreak</td> <td class="desc">Resistance to Do Anything Now (DAN) style jailbreak prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Thinking</span> </a> </td> <td data-sort="97.7"> 97.7% </td> </tr> <tr data-bench="do-not-answer" data-open-source="1"> <td style="text-align:center" data-sort="158">158</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=do-not-answer" class="llm-fu-bench-link" data-bench-id="do-not-answer" title="View models and scores"><strong>Do-Not-Answer</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://huggingface.co/datasets/LibrAI/do-not-answer" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://aclanthology.org/2024.findings-eacl.61" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/Libr-AI/do-not-answer" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety / refusal</td> <td class="desc">Evaluates a model&#39;s ability to refuse unsafe or disallowed requests.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="K2" data-model-version="K2-THINK" data-provider="mbzuai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k2-think"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K2-THINK</span> </a> </td> <td data-sort="88.0"> 88.0% </td> </tr> <tr data-bench="docmath" data-open-source="0"> <td style="text-align:center" data-sort="159">159</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=docmath" class="llm-fu-bench-link" data-bench-id="docmath" title="View models and scores"><strong>DocMath</strong></a> <span class="score-symbol" title="43 scores" aria-label="43 scores">O</span> </td> <td>Document math</td> <td class="desc">Math reasoning on document-based problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="67.6"> 67.6% </td> </tr> <tr data-bench="docvqa" data-open-source="0"> <td style="text-align:center" data-sort="160">160</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=docvqa" class="llm-fu-bench-link" data-bench-id="docvqa" title="View models and scores"><strong>DocVQA</strong></a> <span class="score-symbol" title="130 scores" aria-label="130 scores">O</span> </td> <td>Document understanding (VQA)</td> <td class="desc">Visual question answering over scanned documents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.5" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.5-vl-thinking"> <span class="brand"> <img class="logo no-invert" alt="bytedance logo" data-company="bytedance" data-country="China" src="https://furukama.com/logos/bytedance.svg?v=4e7ed8af" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Seed1.5-VL-Thinking</span> </a> </td> <td data-sort="96.9"> 96.9% </td> </tr> <tr data-bench="dolphin-page" data-open-source="1"> <td style="text-align:center" data-sort="161">161</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=dolphin-page" class="llm-fu-bench-link" data-bench-id="dolphin-page" title="View models and scores"><strong>Dolphin-Page</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Document OCR</td> <td class="desc">Dolphin Page benchmark measuring OCR fidelity and structured extraction on multi-layout documents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Dolphin" data-model-version="Dolphin" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=dolphin-1.5"> <span class="brand"> <img class="logo" alt="microsoft logo" data-company="microsoft" data-country="United States" src="https://furukama.com/logos/microsoft.svg?v=acf5d3c0" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Dolphin 1.5</span> </a> </td> <td data-sort="7.4"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 7.4%</span> </td> </tr> <tr data-bench="dpg-bench" data-open-source="0"> <td style="text-align:center" data-sort="162">162</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=dpg-bench" class="llm-fu-bench-link" data-bench-id="dpg-bench" title="View models and scores"><strong>DPG-Bench</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Text rendering</td> <td class="desc">DPG-Bench score for text rendering in image generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seedream" data-model-version="Seedream" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seedream-4.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seedream 4.5</span> </a> </td> <td data-sort="88.6"> 88.6% </td> </tr> <tr data-bench="drop" data-open-source="1"> <td style="text-align:center" data-sort="163">163</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=drop" class="llm-fu-bench-link" data-bench-id="drop" title="View models and scores"><strong>DROP</strong></a> <span class="score-symbol" title="73 scores" aria-label="73 scores">O</span> <a href="https://allenai.org/data/drop" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Reading &#43; reasoning</td> <td class="desc">Discrete reasoning over paragraphs (addition, counting, comparisons).</td> <td data-sort="2"> <span class="stars" title="Usage: 7/39; GitHub>100: no; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-instruct"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi K2 Instruct</span> </a> </td> <td data-sort="93.5"> 93.5% </td> </tr> <tr data-bench="dude" data-open-source="0"> <td style="text-align:center" data-sort="164">164</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=dude" class="llm-fu-bench-link" data-bench-id="dude" title="View models and scores"><strong>DUDE</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Multimodal long-context</td> <td class="desc">Long-context multimodal understanding benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="70.1"> 70.1% </td> </tr> <tr data-bench="dynamath" data-open-source="1"> <td style="text-align:center" data-sort="165">165</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=dynamath" class="llm-fu-bench-link" data-bench-id="dynamath" title="View models and scores"><strong>DynaMath</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> <a href="https://dynamath.github.io/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://dynamath.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/pdf/2411.00836" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/DynaMath/DynaMath" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Math reasoning (video)</td> <td class="desc">Dynamic/video-based mathematical reasoning evaluating temporal and visual understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-27b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-27B</span> </a> </td> <td data-sort="87.7"> 87.7% </td> </tr> <tr data-bench="economically-important-tasks" data-open-source="0"> <td style="text-align:center" data-sort="166">166</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=economically-important-tasks" class="llm-fu-bench-link" data-bench-id="economically-important-tasks" title="View models and scores"><strong>Economically important tasks</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Industry QA (cross-domain)</td> <td class="desc">Evaluation suite of real-world, economically impactful tasks across key industries and workflows.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="47.1"> 47.1% </td> </tr> <tr data-bench="education-bench" data-open-source="0"> <td style="text-align:center" data-sort="167">167</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=education-bench" class="llm-fu-bench-link" data-bench-id="education-bench" title="View models and scores"><strong>Education</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Economics/education</td> <td class="desc">Education field evaluation (economically valuable tasks).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="60.8"> 60.8% </td> </tr> <tr data-bench="egoschema" data-open-source="1"> <td style="text-align:center" data-sort="168">168</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=egoschema" class="llm-fu-bench-link" data-bench-id="egoschema" title="View models and scores"><strong>EgoSchema</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>Egocentric video QA</td> <td class="desc">EgoSchema validation accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2-vl-72b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2-VL 72B Instruct</span> </a> </td> <td data-sort="77.9"> 77.9% </td> </tr> <tr data-bench="egotempo" data-open-source="0"> <td style="text-align:center" data-sort="169">169</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=egotempo" class="llm-fu-bench-link" data-bench-id="egotempo" title="View models and scores"><strong>EgoTempo</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Egocentric temporal reasoning</td> <td class="desc">Egocentric video temporal reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="67.0"> 67.0% </td> </tr> <tr data-bench="eifbench" data-open-source="0"> <td style="text-align:center" data-sort="170">170</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=eifbench" class="llm-fu-bench-link" data-bench-id="eifbench" title="View models and scores"><strong>EIFBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Instruction following</td> <td class="desc">Complex instruction-following benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="66.7"> 66.7% </td> </tr> <tr data-bench="embspatialbench" data-open-source="1"> <td style="text-align:center" data-sort="171">171</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=embspatialbench" class="llm-fu-bench-link" data-bench-id="embspatialbench" title="View models and scores"><strong>EmbSpatialBench</strong></a> <span class="score-symbol" title="27 scores" aria-label="27 scores">O</span> </td> <td>Spatial understanding</td> <td class="desc">Embodied spatial understanding benchmark evaluating navigation and localization.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="84.5"> 84.5% </td> </tr> <tr data-bench="emma" data-open-source="0"> <td style="text-align:center" data-sort="172">172</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=emma" class="llm-fu-bench-link" data-bench-id="emma" title="View models and scores"><strong>EMMA</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Multimodal reasoning</td> <td class="desc">EMMA benchmark for multimodal reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="66.5"> 66.5% </td> </tr> <tr data-bench="enamel" data-open-source="1"> <td style="text-align:center" data-sort="173">173</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=enamel" class="llm-fu-bench-link" data-bench-id="enamel" title="View models and scores"><strong>Enamel</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Composite capability</td> <td class="desc">Composite capability benchmark capturing broad model performance (Enamel score).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Rnj" data-model-version="Rnj 1" data-provider="essential"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=rnj-1"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Rnj-1</span> </a> </td> <td data-sort="49.0"> 49.0% </td> </tr> <tr data-bench="enconda-bench" data-open-source="1"> <td style="text-align:center" data-sort="174">174</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=enconda-bench" class="llm-fu-bench-link" data-bench-id="enconda-bench" title="View models and scores"><strong>EnConda-Bench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Code editing</td> <td class="desc">English code editing benchmark for applying conditional modifications.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Youtu-LLM" data-model-version="Youtu-LLM" data-provider="tencent"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=youtu-llm-2b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Youtu-LLM-2B</span> </a> </td> <td data-sort="21.5"> 21.5% </td> </tr> <tr data-bench="encyclo-k" data-open-source="0"> <td style="text-align:center" data-sort="175">175</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=encyclo-k" class="llm-fu-bench-link" data-bench-id="encyclo-k" title="View models and scores"><strong>Encyclo-K</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Encyclopedic knowledge</td> <td class="desc">Encyclopedic knowledge evaluation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 2.0" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed2.0-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed2.0 Pro</span> </a> </td> <td data-sort="65.7"> 65.7% </td> </tr> <tr data-bench="enigmaval" data-open-source="0"> <td style="text-align:center" data-sort="176">176</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=enigmaval" class="llm-fu-bench-link" data-bench-id="enigmaval" title="View models and scores"><strong>EnigmaEval</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Challenging puzzles</td> <td class="desc">Challenging puzzle benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="17.8"> 17.8% </td> </tr> <tr data-bench="enterprise-rag" data-open-source="1"> <td style="text-align:center" data-sort="177">177</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=enterprise-rag" class="llm-fu-bench-link" data-bench-id="enterprise-rag" title="View models and scores"><strong>Enterprise RAG</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Retrieval-augmented generation</td> <td class="desc">Enterprise retrieval-augmented generation evaluation covering internal knowledge bases.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Apriel" data-model-version="Apriel Nemotron" data-provider="servicenow"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=apriel-nemotron-15b-thinker"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Apriel Nemotron 15B Thinker</span> </a> </td> <td data-sort="69.2"> 69.2% </td> </tr> <tr data-bench="eqbench" data-open-source="1"> <td style="text-align:center" data-sort="178">178</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=eqbench" class="llm-fu-bench-link" data-bench-id="eqbench" title="View models and scores"><strong>EQ-Bench</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> <a href="https://eqbench.com/eqbench-v2.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://eqbench.com/eqbench-v2.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2312.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/EQ-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Reasoning</td> <td class="desc">General reasoning benchmark assessing equation/logic capabilities.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="352">352</td> <td class="leader-cell" data-model-family="Jan" data-model-version="Jan v1" data-provider="menlo"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=jan-v1-2509"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Jan v1 2509</span> </a> </td> <td data-sort="85.0"> 85.0% </td> </tr> <tr data-bench="eqbench-3" data-open-source="1"> <td style="text-align:center" data-sort="179">179</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=eqbench-3" class="llm-fu-bench-link" data-bench-id="eqbench-3" title="View models and scores"><strong>EQ-Bench 3</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://eqbench.com/index.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://eqbench.com/index.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2312.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/EQ-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Emotional intelligence (roleplay)</td> <td class="desc">A benchmark measuring emotional intelligence in challenging roleplays, judged by Sonnet 3.7.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="21">21</td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-instruct"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi K2 Instruct</span> </a> </td> <td data-sort="1555"> 1555 </td> </tr> <tr data-bench="erqa" data-open-source="0"> <td style="text-align:center" data-sort="180">180</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=erqa" class="llm-fu-bench-link" data-bench-id="erqa" title="View models and scores"><strong>ERQA</strong></a> <span class="score-symbol" title="49 scores" aria-label="49 scores">O</span> </td> <td>Spatial reasoning</td> <td class="desc">Spatial recognition and reasoning QA benchmark (ERQA).</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-flash"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Flash</span> </a> </td> <td data-sort="71.0"> 71.0% </td> </tr> <tr data-bench="evalperf" data-open-source="0"> <td style="text-align:center" data-sort="181">181</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=evalperf" class="llm-fu-bench-link" data-bench-id="evalperf" title="View models and scores"><strong>EvalPerf</strong></a> <span class="score-symbol" title="34 scores" aria-label="34 scores">O</span> <a href="https://evalplus.github.io/evalperf.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://evalplus.github.io/evalperf.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://openreview.net/forum?id=IBCBMeAhmC" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/evalplus/evalplus" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Code evaluation performance</td> <td class="desc">Measures performance of LLM code evaluation, including runtime, memory, and efficiency metrics.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4o" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4o-2024-08-06"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4o (2024-08-06)</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="evalplus" data-open-source="0"> <td style="text-align:center" data-sort="182">182</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=evalplus" class="llm-fu-bench-link" data-bench-id="evalplus" title="View models and scores"><strong>EvalPlus</strong></a> <span class="score-symbol" title="27 scores" aria-label="27 scores">O</span> <a href="https://evalplus.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/evalplus/evalplus" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">Aggregated code evaluation suite from EvalPlus.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1577">1577</td> <td class="leader-cell" data-model-family="o1" data-model-version="o1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o1-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o1 Mini</span> </a> </td> <td data-sort="89.0"> 89.0% </td> </tr> <tr data-bench="evg" data-open-source="1"> <td style="text-align:center" data-sort="183">183</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=evg" class="llm-fu-bench-link" data-bench-id="evg" title="View models and scores"><strong>EVG</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Document OCR</td> <td class="desc">EVG document OCR benchmark evaluating recognition accuracy and layout extraction.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Dolphin" data-model-version="Dolphin" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=dolphin-1.5"> <span class="brand"> <img class="logo" alt="microsoft logo" data-company="microsoft" data-country="United States" src="https://furukama.com/logos/microsoft.svg?v=acf5d3c0" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Dolphin 1.5</span> </a> </td> <td data-sort="3.0"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 3.0%</span> </td> </tr> <tr data-bench="execute" data-open-source="1"> <td style="text-align:center" data-sort="184">184</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=execute" class="llm-fu-bench-link" data-bench-id="execute" title="View models and scores"><strong>EXECUTE</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multilingual character tasks</td> <td class="desc">Multilingual character-level evaluation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Bolmo" data-model-version="Bolmo 7B" data-provider="allenai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=bolmo-7b"> <span class="brand"> <img class="logo" alt="allenai logo" data-company="allenai" data-country="United States" src="https://furukama.com/logos/allenai.svg?v=27b5cbbf" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Bolmo 7B</span> </a> </td> <td data-sort="71.6"> 71.6% </td> </tr> <tr data-bench="facts-benchmark-suite" data-open-source="0"> <td style="text-align:center" data-sort="185">185</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=facts-benchmark-suite" class="llm-fu-bench-link" data-bench-id="facts-benchmark-suite" title="View models and scores"><strong>FACTS Benchmark Suite</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Held out internal grounding, parametric, MM, and search retrieval benchmarks</td> <td class="desc">Comprehensive factuality benchmark suite covering held-out internal grounding, parametric knowledge, multimodal understanding, and search retrieval benchmarks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="70.5"> 70.5% </td> </tr> <tr data-bench="facts-grounding" data-open-source="1"> <td style="text-align:center" data-sort="186">186</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=facts-grounding" class="llm-fu-bench-link" data-bench-id="facts-grounding" title="View models and scores"><strong>FACTS Grounding</strong></a> <span class="score-symbol" title="24 scores" aria-label="24 scores">O</span> <a href="https://www.kaggle.com/benchmarks/google/facts-grounding" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://goo.gle/FACTS_paper" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Grounding / factuality</td> <td class="desc">Grounded factuality benchmark evaluating model alignment with source facts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-instruct"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi K2 Instruct</span> </a> </td> <td data-sort="88.5"> 88.5% </td> </tr> <tr data-bench="factscore" data-open-source="0"> <td style="text-align:center" data-sort="187">187</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=factscore" class="llm-fu-bench-link" data-bench-id="factscore" title="View models and scores"><strong>FActScore</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Hallucination rate on open-source prompts</td> <td class="desc">Measures hallucination rate on an open-source prompt suite; lower is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="1.0"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 1.0%</span> </td> </tr> <tr data-bench="faithjudge-1-hallu" data-open-source="1"> <td style="text-align:center" data-sort="188">188</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faithjudge-1-hallu" class="llm-fu-bench-link" data-bench-id="faithjudge-1-hallu" title="View models and scores"><strong>FaithJudge (1-Hallu.)</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Hallucination detection</td> <td class="desc">FaithJudge hallucination rate with 1-hallucination metric (lower is better).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Moonlight" data-model-version="Moonlight" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=moonlight-instruct"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Moonlight-Instruct</span> </a> </td> <td data-sort="56.0"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 56.0%</span> </td> </tr> <tr data-bench="faix-agent" data-open-source="0"> <td style="text-align:center" data-sort="189">189</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-agent" class="llm-fu-bench-link" data-bench-id="faix-agent" title="View models and scores"><strong>Meta Score Agent</strong></a> <span class="score-symbol" title="76 scores" aria-label="76 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="faix-code" data-open-source="0"> <td style="text-align:center" data-sort="190">190</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-code" class="llm-fu-bench-link" data-bench-id="faix-code" title="View models and scores"><strong>Meta Score Code</strong></a> <span class="score-symbol" title="163 scores" aria-label="163 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="faix-math" data-open-source="1"> <td style="text-align:center" data-sort="191">191</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-math" class="llm-fu-bench-link" data-bench-id="faix-math" title="View models and scores"><strong>Meta Score Math</strong></a> <span class="score-symbol" title="175 scores" aria-label="175 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-235b-a22b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-235B-A22B Instruct</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="faix-ocr" data-open-source="0"> <td style="text-align:center" data-sort="192">192</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-ocr" class="llm-fu-bench-link" data-bench-id="faix-ocr" title="View models and scores"><strong>Meta Score OCR</strong></a> <span class="score-symbol" title="82 scores" aria-label="82 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o3" data-model-version="o3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o3-low"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o3 (Low)</span> </a> </td> <td data-sort="80.0"> 80.0% </td> </tr> <tr data-bench="faix-safety" data-open-source="1"> <td style="text-align:center" data-sort="193">193</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-safety" class="llm-fu-bench-link" data-bench-id="faix-safety" title="View models and scores"><strong>Meta Score Safety</strong></a> <span class="score-symbol" title="53 scores" aria-label="53 scores">O</span> </td> <td>Composite safety index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Granite" data-model-version="Granite 3.3" data-provider="ibm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=granite-3.3-8b-instruct"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Granite 3.3 8B Instruct</span> </a> </td> <td data-sort="70.0"> 70.0% </td> </tr> <tr data-bench="faix-stem" data-open-source="1"> <td style="text-align:center" data-sort="194">194</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-stem" class="llm-fu-bench-link" data-bench-id="faix-stem" title="View models and scores"><strong>Meta Score STEM</strong></a> <span class="score-symbol" title="226 scores" aria-label="226 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-235b-a22b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-235B-A22B Instruct</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="faix-text" data-open-source="0"> <td style="text-align:center" data-sort="195">195</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-text" class="llm-fu-bench-link" data-bench-id="faix-text" title="View models and scores"><strong>Meta Score Text</strong></a> <span class="score-symbol" title="197 scores" aria-label="197 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="85.7"> 85.7% </td> </tr> <tr data-bench="faix-visual" data-open-source="0"> <td style="text-align:center" data-sort="196">196</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-visual" class="llm-fu-bench-link" data-bench-id="faix-visual" title="View models and scores"><strong>Meta Score Visual</strong></a> <span class="score-symbol" title="104 scores" aria-label="104 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="faix-writing" data-open-source="1"> <td style="text-align:center" data-sort="197">197</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=faix-writing" class="llm-fu-bench-link" data-bench-id="faix-writing" title="View models and scores"><strong>Meta Score Writing</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> </td> <td>Composite capability index</td> <td class="desc"></td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen-3-235b-a22b-instruct-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3 235B A22B Instruct 2507</span> </a> </td> <td data-sort="60.0"> 60.0% </td> </tr> <tr data-bench="figqa" data-open-source="0"> <td style="text-align:center" data-sort="198">198</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=figqa" class="llm-fu-bench-link" data-bench-id="figqa" title="View models and scores"><strong>FigQA</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Figure understanding and QA</td> <td class="desc">Figure question answering benchmark evaluating visual reasoning over scientific figures and diagrams.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4.1" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4.1-thinking"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4.1 (Thinking)</span> </a> </td> <td data-sort="34.0"> 34.0% </td> </tr> <tr data-bench="finance-reasoning" data-open-source="1"> <td style="text-align:center" data-sort="199">199</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=finance-reasoning" class="llm-fu-bench-link" data-bench-id="finance-reasoning" title="View models and scores"><strong>FinanceReasoning</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Financial reasoning</td> <td class="desc">Financial reasoning benchmark evaluating quantitative and qualitative finance problem solving.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ling" data-model-version="Ling 1T" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ling-1t"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ling 1T</span> </a> </td> <td data-sort="87.5"> 87.5% </td> </tr> <tr data-bench="financeagent" data-open-source="0"> <td style="text-align:center" data-sort="200">200</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=financeagent" class="llm-fu-bench-link" data-bench-id="financeagent" title="View models and scores"><strong>FinanceAgent</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Agentic finance tasks</td> <td class="desc">Interactive financial agent benchmark requiring multi-step tool use.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="60.7"> 60.7% </td> </tr> <tr data-bench="financeagent-v1.1" data-open-source="0"> <td style="text-align:center" data-sort="201">201</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=financeagent-v1.1" class="llm-fu-bench-link" data-bench-id="financeagent-v1.1" title="View models and scores"><strong>FinanceAgent v1.1</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Agentic finance tasks</td> <td class="desc">Finance Agent v1.1 benchmark for interactive financial agent evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.6</span> </a> </td> <td data-sort="63.3"> 63.3% </td> </tr> <tr data-bench="financebench-fulldoc" data-open-source="0"> <td style="text-align:center" data-sort="202">202</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=financebench-fulldoc" class="llm-fu-bench-link" data-bench-id="financebench-fulldoc" title="View models and scores"><strong>FinanceBench (FullDoc)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Finance QA</td> <td class="desc">FinanceBench full-document question answering benchmark requiring long-context financial understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Jamba" data-model-version="Jamba Mini" data-provider="ai21"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=jamba-mini-1.6"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Jamba Mini 1.6</span> </a> </td> <td data-sort="45.4"> 45.4% </td> </tr> <tr data-bench="finsearchcomp" data-open-source="0"> <td style="text-align:center" data-sort="203">203</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=finsearchcomp" class="llm-fu-bench-link" data-bench-id="finsearchcomp" title="View models and scores"><strong>FinSearchComp</strong></a> <span class="score-symbol" title="27 scores" aria-label="27 scores">O</span> <a href="https://randomtutu.github.io/FinSearchComp/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://randomtutu.github.io/FinSearchComp/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/pdf/2509.13160" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Financial retrieval</td> <td class="desc">Financial search and comprehension benchmark measuring retrieval grounded reasoning over financial content.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4</span> </a> </td> <td data-sort="68.9"> 68.9% </td> </tr> <tr data-bench="finsearchcomp-cn" data-open-source="0"> <td style="text-align:center" data-sort="204">204</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=finsearchcomp-cn" class="llm-fu-bench-link" data-bench-id="finsearchcomp-cn" title="View models and scores"><strong>FinSearchComp-CN</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>Financial retrieval (Chinese)</td> <td class="desc">Chinese financial search and comprehension benchmark measuring retrieval-grounded reasoning over regional financial content.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Doubao" data-model-version="Doubao 1.5" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=doubao-1.5-vision-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">doubao-1-5-vision-pro</span> </a> </td> <td data-sort="54.2"> 54.2% </td> </tr> <tr data-bench="finsearchcomp-t2t3" data-open-source="0"> <td style="text-align:center" data-sort="205">205</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=finsearchcomp-t2t3" class="llm-fu-bench-link" data-bench-id="finsearchcomp-t2t3" title="View models and scores"><strong>FinSearchComp (T2&amp;T3)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Finance search</td> <td class="desc">Finance search competition tasks (tracks T2 and T3).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="64.5"> 64.5% </td> </tr> <tr data-bench="flame-react-eval" data-open-source="1"> <td style="text-align:center" data-sort="206">206</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=flame-react-eval" class="llm-fu-bench-link" data-bench-id="flame-react-eval" title="View models and scores"><strong>Flame-React-Eval</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Frontend coding</td> <td class="desc">Front-end React coding tasks and evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.6V</span> </a> </td> <td data-sort="86.3"> 86.3% </td> </tr> <tr data-bench="flores" data-open-source="1"> <td style="text-align:center" data-sort="207">207</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=flores" class="llm-fu-bench-link" data-bench-id="flores" title="View models and scores"><strong>Flores</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> <a href="https://github.com/facebookresearch/flores" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/facebookresearch/flores" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Machine translation (multilingual)</td> <td class="desc">FLORES multilingual translation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="EuroLLM" data-model-version="EuroLLM" data-provider="eurollm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=eurollm-22b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">EuroLLM-22B</span> </a> </td> <td data-sort="88.9"> 88.9% </td> </tr> <tr data-bench="fox-page-cn" data-open-source="1"> <td style="text-align:center" data-sort="208">208</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=fox-page-cn" class="llm-fu-bench-link" data-bench-id="fox-page-cn" title="View models and scores"><strong>Fox-Page-cn</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Document OCR (Chinese)</td> <td class="desc">Fox Page benchmark evaluating OCR accuracy and layout understanding on Chinese document pages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Dolphin" data-model-version="Dolphin" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=dolphin-1.5"> <span class="brand"> <img class="logo" alt="microsoft logo" data-company="microsoft" data-country="United States" src="https://furukama.com/logos/microsoft.svg?v=acf5d3c0" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Dolphin 1.5</span> </a> </td> <td data-sort="0.8"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 0.8%</span> </td> </tr> <tr data-bench="fox-page-en" data-open-source="1"> <td style="text-align:center" data-sort="209">209</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=fox-page-en" class="llm-fu-bench-link" data-bench-id="fox-page-en" title="View models and scores"><strong>Fox-Page-en</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Document OCR (English)</td> <td class="desc">Fox Page benchmark evaluating OCR accuracy and layout understanding on English document pages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Dolphin" data-model-version="Dolphin" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=dolphin-1.5"> <span class="brand"> <img class="logo" alt="microsoft logo" data-company="microsoft" data-country="United States" src="https://furukama.com/logos/microsoft.svg?v=acf5d3c0" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Dolphin 1.5</span> </a> </td> <td data-sort="0.7"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 0.7%</span> </td> </tr> <tr data-bench="frames" data-open-source="0"> <td style="text-align:center" data-sort="210">210</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=frames" class="llm-fu-bench-link" data-bench-id="frames" title="View models and scores"><strong>FRAMES</strong></a> <span class="score-symbol" title="41 scores" aria-label="41 scores">O</span> </td> <td>Interactive reasoning</td> <td class="desc">Frame-based interactive reasoning and dialogue benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Tongyi" data-model-version="Tongyi" data-provider="tongyi"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=tongyi-deepresearch"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Tongyi DeepResearch</span> </a> </td> <td data-sort="90.6"> 90.6% </td> </tr> <tr data-bench="freshqa" data-open-source="1"> <td style="text-align:center" data-sort="211">211</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=freshqa" class="llm-fu-bench-link" data-bench-id="freshqa" title="View models and scores"><strong>FreshQA</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Recency QA</td> <td class="desc">Question answering benchmark emphasizing up-to-date knowledge and recency.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-4b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-4B Thinking 2507</span> </a> </td> <td data-sort="66.9"> 66.9% </td> </tr> <tr data-bench="frontierscience" data-open-source="0"> <td style="text-align:center" data-sort="212">212</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=frontierscience" class="llm-fu-bench-link" data-bench-id="frontierscience" title="View models and scores"><strong>FrontierScience</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Science reasoning</td> <td class="desc">Frontier-level scientific reasoning and QA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="25.2"> 25.2% </td> </tr> <tr data-bench="frontierscience-olympiad" data-open-source="0"> <td style="text-align:center" data-sort="213">213</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=frontierscience-olympiad" class="llm-fu-bench-link" data-bench-id="frontierscience-olympiad" title="View models and scores"><strong>FrontierScience Olympiad</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Science reasoning (olympiad)</td> <td class="desc">Olympiad-level problems from the FrontierScience benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="75.0"> 75.0% </td> </tr> <tr data-bench="frontierscience-research" data-open-source="0"> <td style="text-align:center" data-sort="214">214</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=frontierscience-research" class="llm-fu-bench-link" data-bench-id="frontierscience-research" title="View models and scores"><strong>FrontierScience Research</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Science reasoning (research)</td> <td class="desc">Research-level problems from the FrontierScience benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="25.0"> 25.0% </td> </tr> <tr data-bench="fsc-147" data-open-source="0"> <td style="text-align:center" data-sort="215">215</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=fsc-147" class="llm-fu-bench-link" data-bench-id="fsc-147" title="View models and scores"><strong>FSC-147</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Few-shot counting</td> <td class="desc">Few-shot counting benchmark across 147 categories.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="33.8"> 33.8% </td> </tr> <tr data-bench="fullstackbench" data-open-source="0"> <td style="text-align:center" data-sort="216">216</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=fullstackbench" class="llm-fu-bench-link" data-bench-id="fullstackbench" title="View models and scores"><strong>FullStackBench</strong></a> <span class="score-symbol" title="48 scores" aria-label="48 scores">O</span> </td> <td>Full-stack development</td> <td class="desc">End-to-end web/app development tasks and evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="72.3"> 72.3% </td> </tr> <tr data-bench="fullstackbench-zh" data-open-source="1"> <td style="text-align:center" data-sort="217">217</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=fullstackbench-zh" class="llm-fu-bench-link" data-bench-id="fullstackbench-zh" title="View models and scores"><strong>FullStackBench (zh)</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Full-stack development</td> <td class="desc">Chinese language full-stack development tasks and evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-235b-a22b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-235B-A22B</span> </a> </td> <td data-sort="63.1"> 63.1% </td> </tr> <tr data-bench="gaia" data-open-source="0"> <td style="text-align:center" data-sort="218">218</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gaia" class="llm-fu-bench-link" data-bench-id="gaia" title="View models and scores"><strong>GAIA</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> <a href="https://gaia-benchmark.github.io/leaderboard/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/GAIA-benchmark/GAIA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/GAIA-benchmark/GAIA" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>General AI tasks</td> <td class="desc">Comprehensive benchmark for agentic tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="87.4"> 87.4% </td> </tr> <tr data-bench="gaia-no-file" data-open-source="1"> <td style="text-align:center" data-sort="219">219</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gaia-no-file" class="llm-fu-bench-link" data-bench-id="gaia-no-file" title="View models and scores"><strong>GAIA (no file)</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://gaia-benchmark.github.io/leaderboard/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/GAIA-benchmark/GAIA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/GAIA-benchmark/GAIA" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>General AI tasks</td> <td class="desc">GAIA benchmark subset without file inputs.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step 3.5 Flash" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step-3.5-flash-20260204"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step-3.5 Flash 20260204</span> </a> </td> <td data-sort="84.5"> 84.5% </td> </tr> <tr data-bench="gaia2" data-open-source="1"> <td style="text-align:center" data-sort="220">220</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gaia2" class="llm-fu-bench-link" data-bench-id="gaia2" title="View models and scores"><strong>GAIA 2</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>General agent tasks</td> <td class="desc">Grounded agentic intelligence benchmark version 2 covering multi-tool tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ring" data-model-version="Ring 2.5" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ring-1t-2.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ring-1T-2.5</span> </a> </td> <td data-sort="75.0"> 75.0% </td> </tr> <tr data-bench="gaokao-bench" data-open-source="1"> <td style="text-align:center" data-sort="221">221</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gaokao-bench" class="llm-fu-bench-link" data-bench-id="gaokao-bench" title="View models and scores"><strong>GAOKAO-Bench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Chinese exams</td> <td class="desc">GAOKAO benchmark measuring Chinese college entrance exam performance.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-instruct-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Instruct-2507</span> </a> </td> <td data-sort="94.5"> 94.5% </td> </tr> <tr data-bench="gdpval" data-open-source="0"> <td style="text-align:center" data-sort="222">222</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gdpval" class="llm-fu-bench-link" data-bench-id="gdpval" title="View models and scores"><strong>GDPVal</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> <a href="https://openai.com/index/gdpval/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://huggingface.co/datasets/openai/gdpval" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://cdn.openai.com/pdf/d5eb7428-c4e9-4a33-bd86-86dd4bcf12ce/GDPval.pdf" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>General capability</td> <td class="desc">GDPVal benchmark evaluating broad general capabilities of LLMs across diverse tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="73.5"> 73.5% </td> </tr> <tr data-bench="gdpval-aa-elo" data-open-source="0"> <td style="text-align:center" data-sort="223">223</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gdpval-aa-elo" class="llm-fu-bench-link" data-bench-id="gdpval-aa-elo" title="View models and scores"><strong>GDPVal-AA Elo</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> <a href="https://openai.com/index/gdpval/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Office tasks</td> <td class="desc">GDPVal Artificial Analysis Elo rating for office-style tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.6</span> </a> </td> <td data-sort="1633"> 1633 </td> </tr> <tr data-bench="general-tool-use" data-open-source="0"> <td style="text-align:center" data-sort="224">224</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=general-tool-use" class="llm-fu-bench-link" data-bench-id="general-tool-use" title="View models and scores"><strong>General Tool Use</strong></a> <span class="score-symbol" title="18 scores" aria-label="18 scores">O</span> </td> <td>Tool use</td> <td class="desc">General tool-use benchmark covering web and API tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="78.9"> 78.9% </td> </tr> <tr data-bench="geobench1" data-open-source="1"> <td style="text-align:center" data-sort="225">225</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=geobench1" class="llm-fu-bench-link" data-bench-id="geobench1" title="View models and scores"><strong>GeoBench1</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Geospatial reasoning</td> <td class="desc">Geospatial visual QA and reasoning (set 1).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.5V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.5v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.5V</span> </a> </td> <td data-sort="79.7"> 79.7% </td> </tr> <tr data-bench="global-mgsm" data-open-source="1"> <td style="text-align:center" data-sort="226">226</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=global-mgsm" class="llm-fu-bench-link" data-bench-id="global-mgsm" title="View models and scores"><strong>GlobalMGSM</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math (multilingual)</td> <td class="desc">Global multilingual grade school math word problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-4b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-4B</span> </a> </td> <td data-sort="60.9"> 60.9% </td> </tr> <tr data-bench="global-mmlu" data-open-source="1"> <td style="text-align:center" data-sort="227">227</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=global-mmlu" class="llm-fu-bench-link" data-bench-id="global-mmlu" title="View models and scores"><strong>Global-MMLU</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Multi-domain knowledge (global)</td> <td class="desc">Full Global-MMLU evaluation across diverse languages and regions.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.2-exp"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.2-Exp</span> </a> </td> <td data-sort="82.0"> 82.0% </td> </tr> <tr data-bench="global-mmlu-lite" data-open-source="0"> <td style="text-align:center" data-sort="228">228</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=global-mmlu-lite" class="llm-fu-bench-link" data-bench-id="global-mmlu-lite" title="View models and scores"><strong>Global-MMLU-Lite</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> <a href="https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multi-domain knowledge (global)</td> <td class="desc">Lightweight global variant of MMLU covering diverse languages and regions.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="89.2"> 89.2% </td> </tr> <tr data-bench="global-piqa" data-open-source="0"> <td style="text-align:center" data-sort="229">229</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=global-piqa" class="llm-fu-bench-link" data-bench-id="global-piqa" title="View models and scores"><strong>Global PIQA</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Commonsense reasoning across 100 Languages and Cultures</td> <td class="desc">Physical commonsense reasoning benchmark spanning 100 languages and diverse cultural contexts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-flash"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Flash</span> </a> </td> <td data-sort="95.6"> 95.6% </td> </tr> <tr data-bench="gorilla-api-bench" data-open-source="1"> <td style="text-align:center" data-sort="230">230</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gorilla-api-bench" class="llm-fu-bench-link" data-bench-id="gorilla-api-bench" title="View models and scores"><strong>Gorilla Benchmark API Bench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://gorilla.cs.berkeley.edu/benchmark" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Tool use</td> <td class="desc">Gorilla API Bench tool-use evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B</span> </a> </td> <td data-sort="35.3"> 35.3% </td> </tr> <tr data-bench="gpqa" data-open-source="0"> <td style="text-align:center" data-sort="231">231</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gpqa" class="llm-fu-bench-link" data-bench-id="gpqa" title="View models and scores"><strong>GPQA</strong></a> <span class="score-symbol" title="173 scores" aria-label="173 scores">O</span> <a href="https://gpqa.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2311.12022" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/idavidrein/gpqa" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Graduate-level QA</td> <td class="desc">Graduate-level question answering evaluating advanced reasoning.</td> <td data-sort="3"> <span class="stars" title="Usage: 7/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="406">406</td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-thinking"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Thinking</span> </a> </td> <td data-sort="92.4"> 92.4% </td> </tr> <tr data-bench="gpqa-diamond" data-open-source="0"> <td style="text-align:center" data-sort="232">232</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gpqa-diamond" class="llm-fu-bench-link" data-bench-id="gpqa-diamond" title="View models and scores"><strong>GPQA-diamond</strong></a> <span class="score-symbol" title="393 scores" aria-label="393 scores">O</span> <a href="https://gpqa.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Graduate-level QA</td> <td class="desc">Hard subset of GPQA (diamond level).</td> <td data-sort="3"> <span class="stars" title="Usage: 15/39; GitHub>100: no; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="94.3"> 94.3% </td> </tr> <tr data-bench="graphwalks-bfs" data-open-source="0"> <td style="text-align:center" data-sort="233">233</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=graphwalks-bfs" class="llm-fu-bench-link" data-bench-id="graphwalks-bfs" title="View models and scores"><strong>GraphWalks BFS</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Long-context reasoning</td> <td class="desc">Graph traversal/GraphWalks benchmark (BFS variant) for long-context reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="98.0"> 98.0% </td> </tr> <tr data-bench="graphwalks-parents" data-open-source="0"> <td style="text-align:center" data-sort="234">234</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=graphwalks-parents" class="llm-fu-bench-link" data-bench-id="graphwalks-parents" title="View models and scores"><strong>GraphWalks Parents</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Graph traversal/GraphWalks benchmark (Parents variant) for long-context reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 2.0" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed2.0-lite"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed2.0 Lite</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="gre-math-maj16" data-open-source="1"> <td style="text-align:center" data-sort="235">235</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gre-math-maj16" class="llm-fu-bench-link" data-bench-id="gre-math-maj16" title="View models and scores"><strong>GRE Math maj@16</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Math (standardized tests)</td> <td class="desc">GRE quantitative section evaluated via majority voting over 16 samples.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2-7b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2 7B</span> </a> </td> <td data-sort="58.5"> 58.5% </td> </tr> <tr data-bench="ground-ui-1k" data-open-source="1"> <td style="text-align:center" data-sort="236">236</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ground-ui-1k" class="llm-fu-bench-link" data-bench-id="ground-ui-1k" title="View models and scores"><strong>Ground-UI-1K</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>GUI grounding</td> <td class="desc">Accuracy on the Ground-UI-1K grounding benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-vl-72b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5-VL 72B</span> </a> </td> <td data-sort="85.4"> 85.4% </td> </tr> <tr data-bench="gsm-infinite-hard-128k" data-open-source="1"> <td style="text-align:center" data-sort="237">237</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm-infinite-hard-128k" class="llm-fu-bench-link" data-bench-id="gsm-infinite-hard-128k" title="View models and scores"><strong>GSM-Infinite Hard (128K)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math reasoning</td> <td class="desc">GSM-Infinite Hard benchmark at 128K context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiMo" data-model-version="MiMo V2" data-provider="xiaomi"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mimo-v2-flash-base"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiMo V2 Flash Base</span> </a> </td> <td data-sort="29.0"> 29.0% </td> </tr> <tr data-bench="gsm-infinite-hard-16k" data-open-source="1"> <td style="text-align:center" data-sort="238">238</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm-infinite-hard-16k" class="llm-fu-bench-link" data-bench-id="gsm-infinite-hard-16k" title="View models and scores"><strong>GSM-Infinite Hard (16K)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math reasoning</td> <td class="desc">GSM-Infinite Hard benchmark at 16K context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.2-exp"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.2-Exp</span> </a> </td> <td data-sort="50.4"> 50.4% </td> </tr> <tr data-bench="gsm-infinite-hard-32k" data-open-source="1"> <td style="text-align:center" data-sort="239">239</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm-infinite-hard-32k" class="llm-fu-bench-link" data-bench-id="gsm-infinite-hard-32k" title="View models and scores"><strong>GSM-Infinite Hard (32K)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math reasoning</td> <td class="desc">GSM-Infinite Hard benchmark at 32K context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.2-exp"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.2-Exp</span> </a> </td> <td data-sort="45.2"> 45.2% </td> </tr> <tr data-bench="gsm-infinite-hard-64k" data-open-source="1"> <td style="text-align:center" data-sort="240">240</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm-infinite-hard-64k" class="llm-fu-bench-link" data-bench-id="gsm-infinite-hard-64k" title="View models and scores"><strong>GSM-Infinite Hard (64K)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math reasoning</td> <td class="desc">GSM-Infinite Hard benchmark at 64K context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.1" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.1"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.1</span> </a> </td> <td data-sort="34.7"> 34.7% </td> </tr> <tr data-bench="gsm-plus" data-open-source="1"> <td style="text-align:center" data-sort="241">241</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm-plus" class="llm-fu-bench-link" data-bench-id="gsm-plus" title="View models and scores"><strong>GSM-Plus</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> </td> <td>Math (grade-school, enhanced)</td> <td class="desc">Enhanced GSM-style grade-school math benchmark variant.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LLaDA" data-model-version="LLaDA 2.0" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llada2.0-flash"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">LLaDA2.0 Flash</span> </a> </td> <td data-sort="89.7"> 89.7% </td> </tr> <tr data-bench="gsm-symbolic" data-open-source="1"> <td style="text-align:center" data-sort="242">242</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm-symbolic" class="llm-fu-bench-link" data-bench-id="gsm-symbolic" title="View models and scores"><strong>GSM-Symbolic</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://github.com/kosukearai/gsm-symbolic" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/kosukearai/gsm-symbolic" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Math reasoning</td> <td class="desc">Symbolic reasoning variant of GSM that tests algebraic manipulation and arithmetic with structured problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Granite" data-model-version="Granite 4.0" data-provider="ibm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=granite-4.0-h-small"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Granite-4.0-H-Small</span> </a> </td> <td data-sort="87.4"> 87.4% </td> </tr> <tr data-bench="gsm8k" data-open-source="1"> <td style="text-align:center" data-sort="243">243</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm8k" class="llm-fu-bench-link" data-bench-id="gsm8k" title="View models and scores"><strong>GSM8K</strong></a> <span class="score-symbol" title="301 scores" aria-label="301 scores">O</span> <a href="https://llm-stats.com/benchmarks/gsm8k" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://huggingface.co/datasets/openai/gsm8k" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/openai/grade-school-math" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Math (grade-school)</td> <td class="desc">Grade-school math word problems requiring multi-step reasoning.</td> <td data-sort="3"> <span class="stars" title="Usage: 8/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="1322">1322</td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-instruct"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi K2 Instruct</span> </a> </td> <td data-sort="97.3"> 97.3% </td> </tr> <tr data-bench="gsm8k-de" data-open-source="1"> <td style="text-align:center" data-sort="244">244</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm8k-de" class="llm-fu-bench-link" data-bench-id="gsm8k-de" title="View models and scores"><strong>GSM8K (DE)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Math (grade-school, German)</td> <td class="desc">German translation of the GSM8K grade-school math word problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="0.6"> 0.6% </td> </tr> <tr data-bench="gsm8k-ko" data-open-source="1"> <td style="text-align:center" data-sort="245">245</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm8k-ko" class="llm-fu-bench-link" data-bench-id="gsm8k-ko" title="View models and scores"><strong>GSM8K-Ko</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math (grade-school, Korean)</td> <td class="desc">Korean translation of the GSM8K grade-school math word problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B</span> </a> </td> <td data-sort="88.1"> 88.1% </td> </tr> <tr data-bench="gsm8k-platinum" data-open-source="1"> <td style="text-align:center" data-sort="246">246</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gsm8k-platinum" class="llm-fu-bench-link" data-bench-id="gsm8k-platinum" title="View models and scores"><strong>GSM8K Platinum</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/openai/grade-school-math" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (grade-school, hard)</td> <td class="desc">Harder subset/setting of GSM8K grade-school math problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi Linear" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-linear-base"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-Linear-Base</span> </a> </td> <td data-sort="89.6"> 89.6% </td> </tr> <tr data-bench="gso-benchmark" data-open-source="0"> <td style="text-align:center" data-sort="247">247</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=gso-benchmark" class="llm-fu-bench-link" data-bench-id="gso-benchmark" title="View models and scores"><strong>GSO Benchmark</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> <a href="https://gso-bench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://livecodebench.github.io/gso.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">LiveCodeBench GSO benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o3" data-model-version="o3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o3-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o3-high</span> </a> </td> <td data-sort="8.8"> 8.8% </td> </tr> <tr data-bench="hae-rae-bench" data-open-source="1"> <td style="text-align:center" data-sort="248">248</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hae-rae-bench" class="llm-fu-bench-link" data-bench-id="hae-rae-bench" title="View models and scores"><strong>HAE-RAE Bench</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> <a href="https://huggingface.co/datasets/HAERAE-HUB/HAE_RAE_BENCH" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Korean language understanding</td> <td class="desc">Korean language understanding benchmark evaluating knowledge and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kanana" data-model-version="Kanana 1.5" data-provider="kakao"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kanana-1.5-32.5b-base"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Kanana-1.5-32.5B-Base</span> </a> </td> <td data-sort="90.7"> 90.7% </td> </tr> <tr data-bench="hallusionbench" data-open-source="1"> <td style="text-align:center" data-sort="249">249</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hallusionbench" class="llm-fu-bench-link" data-bench-id="hallusionbench" title="View models and scores"><strong>HallusionBench</strong></a> <span class="score-symbol" title="46 scores" aria-label="46 scores">O</span> <a href="https://hallusionbench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2310.14566" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/FuxiaoLiu/HallusionBench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multimodal hallucination</td> <td class="desc">Benchmark for evaluating hallucination tendencies in multimodal LLMs.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="71.4"> 71.4% </td> </tr> <tr data-bench="harmbench" data-open-source="1"> <td style="text-align:center" data-sort="250">250</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=harmbench" class="llm-fu-bench-link" data-bench-id="harmbench" title="View models and scores"><strong>HarmBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Safety</td> <td class="desc">Harmfulness and safety compliance benchmark across a variety of risky prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Thinking</span> </a> </td> <td data-sort="94.8"> 94.8% </td> </tr> <tr data-bench="harmfulq" data-open-source="1"> <td style="text-align:center" data-sort="251">251</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=harmfulq" class="llm-fu-bench-link" data-bench-id="harmfulq" title="View models and scores"><strong>HarmfulQA</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://huggingface.co/datasets/d-llm/HarmfulQA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2308.09662" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/declare-lab/red-instruct" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety</td> <td class="desc">Harmful question set testing models&#39; ability to avoid unsafe answers.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="104">104</td> <td class="leader-cell" data-model-family="K2" data-model-version="K2-THINK" data-provider="mbzuai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k2-think"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K2-THINK</span> </a> </td> <td data-sort="99.0"> 99.0% </td> </tr> <tr data-bench="healthbench" data-open-source="0"> <td style="text-align:center" data-sort="252">252</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=healthbench" class="llm-fu-bench-link" data-bench-id="healthbench" title="View models and scores"><strong>HealthBench</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Medical QA</td> <td class="desc">Comprehensive medical knowledge and clinical reasoning benchmark across specialties and tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="67.2"> 67.2% </td> </tr> <tr data-bench="healthbench-hard" data-open-source="0"> <td style="text-align:center" data-sort="253">253</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=healthbench-hard" class="llm-fu-bench-link" data-bench-id="healthbench-hard" title="View models and scores"><strong>HealthBench-Hard</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>Medical QA (hard)</td> <td class="desc">Challenging subset of HealthBench focusing on complex, ambiguous clinical cases.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="46.2"> 46.2% </td> </tr> <tr data-bench="healthbench-hard-hallucinations" data-open-source="0"> <td style="text-align:center" data-sort="254">254</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=healthbench-hard-hallucinations" class="llm-fu-bench-link" data-bench-id="healthbench-hard-hallucinations" title="View models and scores"><strong>HealthBench-Hard Hallucinations</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Medical hallucination safety</td> <td class="desc">Measures hallucination and unsafe medical advice under hard clinical scenarios.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="1.6"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 1.6%</span> </td> </tr> <tr data-bench="hellaswag" data-open-source="1"> <td style="text-align:center" data-sort="255">255</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hellaswag" class="llm-fu-bench-link" data-bench-id="hellaswag" title="View models and scores"><strong>HellaSwag</strong></a> <span class="score-symbol" title="183 scores" aria-label="183 scores">O</span> <a href="https://github.com/rowanz/hellaswag" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Commonsense reasoning</td> <td class="desc">Adversarial commonsense sentence completion.</td> <td data-sort="2"> <span class="stars" title="Usage: 5/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="220">220</td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3-base"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3 Base</span> </a> </td> <td data-sort="96.4"> 96.4% </td> </tr> <tr data-bench="hellaswag-de" data-open-source="1"> <td style="text-align:center" data-sort="256">256</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hellaswag-de" class="llm-fu-bench-link" data-bench-id="hellaswag-de" title="View models and scores"><strong>HellaSwag (DE)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Commonsense reasoning (German)</td> <td class="desc">German translation of the HellaSwag commonsense benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="0.7"> 0.7% </td> </tr> <tr data-bench="helmet-longqa" data-open-source="0"> <td style="text-align:center" data-sort="257">257</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=helmet-longqa" class="llm-fu-bench-link" data-bench-id="helmet-longqa" title="View models and scores"><strong>HELMET LongQA</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Long-context QA</td> <td class="desc">Long-context subset of the HELMET benchmark focusing on grounded question answering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Jamba" data-model-version="Jamba Mini" data-provider="ai21"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=jamba-mini-1.6"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Jamba Mini 1.6</span> </a> </td> <td data-sort="46.9"> 46.9% </td> </tr> <tr data-bench="herobench" data-open-source="0"> <td style="text-align:center" data-sort="258">258</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=herobench" class="llm-fu-bench-link" data-bench-id="herobench" title="View models and scores"><strong>HeroBench</strong></a> <span class="score-symbol" title="24 scores" aria-label="24 scores">O</span> <a href="https://arxiv.org/pdf/2508.12782" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-horizon planning</td> <td class="desc">Benchmark for Long-Horizon Planning and Structured Reasoning in Virtual Worlds.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4</span> </a> </td> <td data-sort="91.7"> 91.7% </td> </tr> <tr data-bench="hhem-v2.1" data-open-source="1"> <td style="text-align:center" data-sort="259">259</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hhem-v2.1" class="llm-fu-bench-link" data-bench-id="hhem-v2.1" title="View models and scores"><strong>HHEM v2.1</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> <a href="https://github.com/vectara/hallucination-leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/vectara/hallucination-leaderboard" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Hallucination detection</td> <td class="desc">Hughes Hallucination Evaluation Model (Vectara) — lower is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Finix" data-model-version="Finix S1" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=antgroup-finix-s1-32b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">AntGroup Finix_S1_32b</span> </a> </td> <td data-sort="0.6"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 0.6%</span> </td> </tr> <tr data-bench="hiddenmath" data-open-source="0"> <td style="text-align:center" data-sort="260">260</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hiddenmath" class="llm-fu-bench-link" data-bench-id="hiddenmath" title="View models and scores"><strong>HiddenMath</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Math reasoning</td> <td class="desc">Mathematical reasoning benchmark referenced in recent model cards.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.0" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.0-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.0 Pro</span> </a> </td> <td data-sort="65.2"> 65.2% </td> </tr> <tr data-bench="hle" data-open-source="0"> <td style="text-align:center" data-sort="261">261</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hle" class="llm-fu-bench-link" data-bench-id="hle" title="View models and scores"><strong>HLE</strong></a> <span class="score-symbol" title="215 scores" aria-label="215 scores">O</span> <a href="https://agi.safe.ai/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/hle" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multi-domain reasoning</td> <td class="desc">Challenging LLMs at the frontier of human knowledge.</td> <td data-sort="2"> <span class="stars" title="Usage: 5/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1085">1085</td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-deep-think"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Deep Think</span> </a> </td> <td data-sort="48.4"> 48.4% </td> </tr> <tr data-bench="hle-overconfidence" data-open-source="0"> <td style="text-align:center" data-sort="262">262</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hle-overconfidence" class="llm-fu-bench-link" data-bench-id="hle-overconfidence" title="View models and scores"><strong>HLE Overconfidence</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Overconfidence / safety</td> <td class="desc">Overconfidence rate derived from Humanity&#39;s Last Exam evaluations.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="43.7"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 43.7%</span> </td> </tr> <tr data-bench="hle-text-only" data-open-source="0"> <td style="text-align:center" data-sort="263">263</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hle-text-only" class="llm-fu-bench-link" data-bench-id="hle-text-only" title="View models and scores"><strong>HLE (Text Only)</strong></a> <span class="score-symbol" title="109 scores" aria-label="109 scores">O</span> <a href="https://agi.safe.ai/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/hle" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Advanced reasoning</td> <td class="desc">Humanity&#39;s Last Exam benchmark restricted to text-only inputs.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1085">1085</td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="45.8"> 45.8% </td> </tr> <tr data-bench="hle-verified" data-open-source="0"> <td style="text-align:center" data-sort="264">264</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hle-verified" class="llm-fu-bench-link" data-bench-id="hle-verified" title="View models and scores"><strong>HLE-Verified</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://huggingface.co/datasets/skylenage/HLE-Verified" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multi-domain reasoning</td> <td class="desc">Verified and revised version of Humanity&#39;s Last Exam (HLE) with component-wise verification protocol.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="48.0"> 48.0% </td> </tr> <tr data-bench="hle-vl" data-open-source="0"> <td style="text-align:center" data-sort="265">265</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hle-vl" class="llm-fu-bench-link" data-bench-id="hle-vl" title="View models and scores"><strong>HLE-VL</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Holistic language evaluation (vision-language)</td> <td class="desc">Vision-language HLE benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="36.0"> 36.0% </td> </tr> <tr data-bench="hle-with-tools" data-open-source="0"> <td style="text-align:center" data-sort="266">266</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hle-with-tools" class="llm-fu-bench-link" data-bench-id="hle-with-tools" title="View models and scores"><strong>HLE (With Tools)</strong></a> <span class="score-symbol" title="40 scores" aria-label="40 scores">O</span> <a href="https://agi.safe.ai/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/hle" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Tool-augmented reasoning</td> <td class="desc">Humanity&#39;s Last Exam benchmark evaluated with tool access.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1085">1085</td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="53.1"> 53.1% </td> </tr> <tr data-bench="hmmt" data-open-source="0"> <td style="text-align:center" data-sort="267">267</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hmmt" class="llm-fu-bench-link" data-bench-id="hmmt" title="View models and scores"><strong>HMMT</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://hmmt.org/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">Harvard–MIT Mathematics Tournament problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-pro"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 pro</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="hmmt-2025" data-open-source="0"> <td style="text-align:center" data-sort="268">268</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hmmt-2025" class="llm-fu-bench-link" data-bench-id="hmmt-2025" title="View models and scores"><strong>HMMT 2025</strong></a> <span class="score-symbol" title="117 scores" aria-label="117 scores">O</span> </td> <td>Math (competition)</td> <td class="desc">Harvard–MIT Mathematics Tournament 2025 problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="99.8"> 99.8% </td> </tr> <tr data-bench="hmmt-feb-2025" data-open-source="0"> <td style="text-align:center" data-sort="269">269</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hmmt-feb-2025" class="llm-fu-bench-link" data-bench-id="hmmt-feb-2025" title="View models and scores"><strong>HMMT Feb 2025</strong></a> <span class="score-symbol" title="41 scores" aria-label="41 scores">O</span> <a href="https://hmmt.org/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">Harvard–MIT Mathematics Tournament February 2025 problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="hmmt-nov-2025" data-open-source="0"> <td style="text-align:center" data-sort="270">270</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hmmt-nov-2025" class="llm-fu-bench-link" data-bench-id="hmmt-nov-2025" title="View models and scores"><strong>HMMT Nov 2025</strong></a> <span class="score-symbol" title="51 scores" aria-label="51 scores">O</span> <a href="https://hmmt.org/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">Harvard–MIT Mathematics Tournament November 2025 problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="hotpotqa" data-open-source="1"> <td style="text-align:center" data-sort="271">271</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hotpotqa" class="llm-fu-bench-link" data-bench-id="hotpotqa" title="View models and scores"><strong>HotpotQA</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://hotpotqa.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multi-hop QA</td> <td class="desc">Explainable multi-hop QA with supporting facts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen-3-0.6b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen 3 0.6B</span> </a> </td> <td data-sort="64.0"> 64.0% </td> </tr> <tr data-bench="hrbench4k" data-open-source="1"> <td style="text-align:center" data-sort="272">272</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hrbench4k" class="llm-fu-bench-link" data-bench-id="hrbench4k" title="View models and scores"><strong>HRBench 4K</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Hallucination robustness</td> <td class="desc">Hallucination robustness benchmark with 4K token contexts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-30b-a3b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-30B-A3B Instruct</span> </a> </td> <td data-sort="89.5"> 89.5% </td> </tr> <tr data-bench="hrbench8k" data-open-source="0"> <td style="text-align:center" data-sort="273">273</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hrbench8k" class="llm-fu-bench-link" data-bench-id="hrbench8k" title="View models and scores"><strong>HRBench 8K</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Hallucination robustness</td> <td class="desc">Hallucination robustness benchmark with 8K token contexts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="84.0"> 84.0% </td> </tr> <tr data-bench="hrm8k" data-open-source="1"> <td style="text-align:center" data-sort="274">274</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hrm8k" class="llm-fu-bench-link" data-bench-id="hrm8k" title="View models and scores"><strong>HRM8K</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Korean reasoning</td> <td class="desc">8k-question Korean reasoning and knowledge benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-235b-a22b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-235B-A22B-Thinking-2507</span> </a> </td> <td data-sort="92.0"> 92.0% </td> </tr> <tr data-bench="humaneval" data-open-source="0"> <td style="text-align:center" data-sort="275">275</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=humaneval" class="llm-fu-bench-link" data-bench-id="humaneval" title="View models and scores"><strong>HumanEval</strong></a> <span class="score-symbol" title="279 scores" aria-label="279 scores">O</span> <a href="https://llmdb.com/benchmarks/humaneval" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/openai/human-eval" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2107.03374" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/openai/human-eval" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">Python synthesis problems evaluated by unit tests.</td> <td data-sort="3"> <span class="stars" title="Usage: 8/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="2916">2916</td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro-preview"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro Preview</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="humaneval-plus" data-open-source="0"> <td style="text-align:center" data-sort="276">276</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=humaneval-plus" class="llm-fu-bench-link" data-bench-id="humaneval-plus" title="View models and scores"><strong>HumanEval&#43;</strong></a> <span class="score-symbol" title="298 scores" aria-label="298 scores">O</span> <a href="https://github.com/evalplus/evalplus" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">Extended HumanEval with more tests.</td> <td data-sort="2"> <span class="stars" title="Usage: 2/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1577">1577</td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4</span> </a> </td> <td data-sort="94.5"> 94.5% </td> </tr> <tr data-bench="humaneval-v" data-open-source="1"> <td style="text-align:center" data-sort="277">277</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=humaneval-v" class="llm-fu-bench-link" data-bench-id="humaneval-v" title="View models and scores"><strong>HumanEval-V</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Code generation (vision)</td> <td class="desc">HumanEval variant with visual programming prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step3-VL" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step3-vl-10b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step3-VL-10B</span> </a> </td> <td data-sort="66.0"> 66.0% </td> </tr> <tr data-bench="humaneval-x" data-open-source="1"> <td style="text-align:center" data-sort="278">278</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=humaneval-x" class="llm-fu-bench-link" data-bench-id="humaneval-x" title="View models and scores"><strong>HumanEval-X</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://github.com/THUDM/CodeGeeX" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/THUDM/CodeGeeX" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Code generation (multilingual)</td> <td class="desc">Multilingual code generation benchmark extending HumanEval to multiple programming languages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="TeleChat" data-model-version="TeleChat3" data-provider="teleai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=telechat3-36b-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">TeleChat3-36B-Thinking</span> </a> </td> <td data-sort="92.7"> 92.7% </td> </tr> <tr data-bench="hypersim" data-open-source="0"> <td style="text-align:center" data-sort="279">279</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=hypersim" class="llm-fu-bench-link" data-bench-id="hypersim" title="View models and scores"><strong>Hypersim</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>3D scene understanding</td> <td class="desc">Hypersim benchmark for synthetic indoor scene understanding and reconstruction.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-mini-minimal"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 Mini Minimal</span> </a> </td> <td data-sort="39.3"> 39.3% </td> </tr> <tr data-bench="ifbench" data-open-source="1"> <td style="text-align:center" data-sort="280">280</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ifbench" class="llm-fu-bench-link" data-bench-id="ifbench" title="View models and scores"><strong>IFBench</strong></a> <span class="score-symbol" title="151 scores" aria-label="151 scores">O</span> <a href="https://github.com/allenai/IFBench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Instruction following</td> <td class="desc">Instruction-following benchmark measuring compliance and adherence.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="70">70</td> <td class="leader-cell" data-model-family="Mistral" data-model-version="Mistral 3.2" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mistral-small-3.2-24b-instruct"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mistral Small 3.2 24B Instruct</span> </a> </td> <td data-sort="84.8"> 84.8% </td> </tr> <tr data-bench="ifeval" data-open-source="1"> <td style="text-align:center" data-sort="281">281</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ifeval" class="llm-fu-bench-link" data-bench-id="ifeval" title="View models and scores"><strong>IFEval</strong></a> <span class="score-symbol" title="311 scores" aria-label="311 scores">O</span> <a href="https://github.com/google-research/google-research/tree/master/ifeval" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2311.07911" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/google-research/google-research/tree/master/instruction_following_eval" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Instruction following</td> <td class="desc">Instruction following capability evaluation for LLMs.</td> <td data-sort="3"> <span class="stars" title="Usage: 11/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="36312">36312</td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-27b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-27B</span> </a> </td> <td data-sort="95.0"> 95.0% </td> </tr> <tr data-bench="ifeval-code" data-open-source="1"> <td style="text-align:center" data-sort="282">282</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ifeval-code" class="llm-fu-bench-link" data-bench-id="ifeval-code" title="View models and scores"><strong>IFEval-Code</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Instruction following (code)</td> <td class="desc">Instruction following evaluation for code generation tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-32b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-32B</span> </a> </td> <td data-sort="28.0"> 28.0% </td> </tr> <tr data-bench="ifeval-strict-prompt" data-open-source="1"> <td style="text-align:center" data-sort="283">283</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ifeval-strict-prompt" class="llm-fu-bench-link" data-bench-id="ifeval-strict-prompt" title="View models and scores"><strong>IFEval (Strict Prompt)</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Instruction following</td> <td class="desc">IFEval strict prompt-level accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-8b-non-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-8B Non-Thinking</span> </a> </td> <td data-sort="84.3"> 84.3% </td> </tr> <tr data-bench="img-qa-avg" data-open-source="0"> <td style="text-align:center" data-sort="284">284</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=img-qa-avg" class="llm-fu-bench-link" data-bench-id="img-qa-avg" title="View models and scores"><strong>Image QA Average</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> </td> <td>Image QA (aggregate)</td> <td class="desc">Average of single-image visual question answering benchmarks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="86.2"> 86.2% </td> </tr> <tr data-bench="imo-answerbench" data-open-source="1"> <td style="text-align:center" data-sort="285">285</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=imo-answerbench" class="llm-fu-bench-link" data-bench-id="imo-answerbench" title="View models and scores"><strong>IMO AnswerBench</strong></a> <span class="score-symbol" title="78 scores" aria-label="78 scores">O</span> <a href="https://imobench.github.io/#leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://imobench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2511.01846" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/google-deepmind/superhuman/tree/main/imobench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">Evaluates free-form solutions to International Mathematical Olympiad problems using expert-style grading rubrics.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ring" data-model-version="Ring 2.5" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ring-1t-2.5-heavy-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ring-1T-2.5-heavy-thinking</span> </a> </td> <td data-sort="90.0"> 90.0% </td> </tr> <tr data-bench="include" data-open-source="0"> <td style="text-align:center" data-sort="286">286</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=include" class="llm-fu-bench-link" data-bench-id="include" title="View models and scores"><strong>INCLUDE</strong></a> <span class="score-symbol" title="37 scores" aria-label="37 scores">O</span> </td> <td>Inclusiveness / bias</td> <td class="desc">Evaluates inclusive language use and bias mitigation in model outputs.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="90.5"> 90.5% </td> </tr> <tr data-bench="infoqa" data-open-source="0"> <td style="text-align:center" data-sort="287">287</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=infoqa" class="llm-fu-bench-link" data-bench-id="infoqa" title="View models and scores"><strong>InfoQA</strong></a> <span class="score-symbol" title="47 scores" aria-label="47 scores">O</span> </td> <td>Information-seeking QA</td> <td class="desc">Information retrieval question answering benchmark evaluating factual responses.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="86.9"> 86.9% </td> </tr> <tr data-bench="information-extraction" data-open-source="0"> <td style="text-align:center" data-sort="288">288</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=information-extraction" class="llm-fu-bench-link" data-bench-id="information-extraction" title="View models and scores"><strong>Information Extraction</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Information extraction</td> <td class="desc">Information extraction benchmark for economically valuable fields.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5</span> </a> </td> <td data-sort="46.9"> 46.9% </td> </tr> <tr data-bench="information-processing" data-open-source="0"> <td style="text-align:center" data-sort="289">289</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=information-processing" class="llm-fu-bench-link" data-bench-id="information-processing" title="View models and scores"><strong>Information Processing</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Information processing</td> <td class="desc">Information processing benchmark for economically valuable tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="56.5"> 56.5% </td> </tr> <tr data-bench="infovqa" data-open-source="1"> <td style="text-align:center" data-sort="290">290</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=infovqa" class="llm-fu-bench-link" data-bench-id="infovqa" title="View models and scores"><strong>InfoVQA</strong></a> <span class="score-symbol" title="74 scores" aria-label="74 scores">O</span> </td> <td>Infographic VQA</td> <td class="desc">Visual question answering over infographics requiring reading, counting, and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5 Thinking</span> </a> </td> <td data-sort="92.6"> 92.6% </td> </tr> <tr data-bench="intention-recognition" data-open-source="0"> <td style="text-align:center" data-sort="291">291</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=intention-recognition" class="llm-fu-bench-link" data-bench-id="intention-recognition" title="View models and scores"><strong>Intention Recognition</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Intent recognition</td> <td class="desc">Intent recognition benchmark for practical applications.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="65.3"> 65.3% </td> </tr> <tr data-bench="intphys-2" data-open-source="0"> <td style="text-align:center" data-sort="292">292</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=intphys-2" class="llm-fu-bench-link" data-bench-id="intphys-2" title="View models and scores"><strong>IntPhys 2</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Intuitive physics</td> <td class="desc">Intuitive physics reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-flash"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Flash</span> </a> </td> <td data-sort="63.4"> 63.4% </td> </tr> <tr data-bench="inverse-ifeval" data-open-source="0"> <td style="text-align:center" data-sort="293">293</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=inverse-ifeval" class="llm-fu-bench-link" data-bench-id="inverse-ifeval" title="View models and scores"><strong>Inverse IFEval</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Instruction following (inverse)</td> <td class="desc">Inverse instruction-following evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-flash"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Flash</span> </a> </td> <td data-sort="80.9"> 80.9% </td> </tr> <tr data-bench="isl-osl-8k-16k" data-open-source="1"> <td style="text-align:center" data-sort="294">294</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=isl-osl-8k-16k" class="llm-fu-bench-link" data-bench-id="isl-osl-8k-16k" title="View models and scores"><strong>ISL/OSL 8k/16k</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Throughput</td> <td class="desc">Relative throughput on ISL/OSL 8k/16k context workloads.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nemotron-3-nano-30b-a3b"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Nemotron-3-Nano-30B-A3B</span> </a> </td> <td data-sort="3.3"> 3.3% </td> </tr> <tr data-bench="judgemark-v2-1" data-open-source="0"> <td style="text-align:center" data-sort="295">295</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=judgemark-v2-1" class="llm-fu-bench-link" data-bench-id="judgemark-v2-1" title="View models and scores"><strong>JudgeMark v2.1</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://eqbench.com/judgemark-v2.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>LLM judging ability</td> <td class="desc">A benchmark measuring LLM judging ability.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4</span> </a> </td> <td data-sort="82.0"> 82.0% </td> </tr> <tr data-bench="kgc-safety" data-open-source="1"> <td style="text-align:center" data-sort="296">296</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kgc-safety" class="llm-fu-bench-link" data-bench-id="kgc-safety" title="View models and scores"><strong>KGC-Safety</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Safety (Korean)</td> <td class="desc">Korean safety benchmark evaluating harmfulness and compliance.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="K-EXAONE" data-model-version="K-EXAONE" data-provider="lg"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k-exaone"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K-EXAONE</span> </a> </td> <td data-sort="96.1"> 96.1% </td> </tr> <tr data-bench="kk-4-people" data-open-source="1"> <td style="text-align:center" data-sort="297">297</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kk-4-people" class="llm-fu-bench-link" data-bench-id="kk-4-people" title="View models and scores"><strong>KK-4 People</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Working memory (4 people)</td> <td class="desc">Keep/kill working-memory benchmark with 4 people entities.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="K2" data-model-version="K2 V2" data-provider="mbzuai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k2-v2"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K2-V2</span> </a> </td> <td data-sort="92.9"> 92.9% </td> </tr> <tr data-bench="kk-8-people" data-open-source="1"> <td style="text-align:center" data-sort="298">298</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kk-8-people" class="llm-fu-bench-link" data-bench-id="kk-8-people" title="View models and scores"><strong>KK-8 People</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Working memory (8 people)</td> <td class="desc">Keep/kill working-memory benchmark with 8 people entities.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="K2" data-model-version="K2 V2" data-provider="mbzuai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k2-v2"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K2-V2</span> </a> </td> <td data-sort="82.8"> 82.8% </td> </tr> <tr data-bench="kmmlu" data-open-source="1"> <td style="text-align:center" data-sort="299">299</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kmmlu" class="llm-fu-bench-link" data-bench-id="kmmlu" title="View models and scores"><strong>KMMLU</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> <a href="https://huggingface.co/datasets/HAERAE-HUB/KMMLU" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Korean knowledge</td> <td class="desc">Korean Massive Multitask Language Understanding benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.1" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.1"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.1</span> </a> </td> <td data-sort="78.7"> 78.7% </td> </tr> <tr data-bench="kmmlu-pro" data-open-source="0"> <td style="text-align:center" data-sort="300">300</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kmmlu-pro" class="llm-fu-bench-link" data-bench-id="kmmlu-pro" title="View models and scores"><strong>KMMLU-Pro</strong></a> <span class="score-symbol" title="59 scores" aria-label="59 scores">O</span> <a href="https://huggingface.co/datasets/HAERAE-HUB/KMMLU" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multilingual knowledge</td> <td class="desc">Korean Multilingual Massive Multitask Language Understanding Pro</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o1" data-model-version="o1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o1"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o1</span> </a> </td> <td data-sort="77.5"> 77.5% </td> </tr> <tr data-bench="kmmlu-redux" data-open-source="0"> <td style="text-align:center" data-sort="301">301</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kmmlu-redux" class="llm-fu-bench-link" data-bench-id="kmmlu-redux" title="View models and scores"><strong>KMMLU-Redux</strong></a> <span class="score-symbol" title="54 scores" aria-label="54 scores">O</span> <a href="https://huggingface.co/datasets/HAERAE-HUB/KMMLU" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2507.08924" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Multilingual knowledge</td> <td class="desc">Redux variant of KMMLU benchmark</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o1" data-model-version="o1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o1"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o1</span> </a> </td> <td data-sort="81.1"> 81.1% </td> </tr> <tr data-bench="ko-longbench" data-open-source="1"> <td style="text-align:center" data-sort="302">302</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ko-longbench" class="llm-fu-bench-link" data-bench-id="ko-longbench" title="View models and scores"><strong>Ko-LongBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Korean long-context</td> <td class="desc">Long-context understanding benchmark in Korean.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.2-thinking"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.2-Thinking</span> </a> </td> <td data-sort="87.9"> 87.9% </td> </tr> <tr data-bench="kobolt" data-open-source="1"> <td style="text-align:center" data-sort="303">303</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kobolt" class="llm-fu-bench-link" data-bench-id="kobolt" title="View models and scores"><strong>KoBALT</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Korean knowledge</td> <td class="desc">Korean benchmark for knowledge and language understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.2-thinking"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.2-Thinking</span> </a> </td> <td data-sort="62.7"> 62.7% </td> </tr> <tr data-bench="komt-bench" data-open-source="1"> <td style="text-align:center" data-sort="304">304</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=komt-bench" class="llm-fu-bench-link" data-bench-id="komt-bench" title="View models and scores"><strong>KoMT-Bench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Korean chat ability</td> <td class="desc">Korean multi-turn chat evaluation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-instruct-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Instruct-2507</span> </a> </td> <td data-sort="8.5"> 8.5% </td> </tr> <tr data-bench="kor-bench" data-open-source="0"> <td style="text-align:center" data-sort="305">305</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kor-bench" class="llm-fu-bench-link" data-bench-id="kor-bench" title="View models and scores"><strong>KOR-Bench</strong></a> <span class="score-symbol" title="24 scores" aria-label="24 scores">O</span> </td> <td>Reasoning</td> <td class="desc">Comprehensive reasoning benchmark spanning diverse domains and cognitive skills.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="77.4"> 77.4% </td> </tr> <tr data-bench="korbench" data-open-source="0"> <td style="text-align:center" data-sort="306">306</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=korbench" class="llm-fu-bench-link" data-bench-id="korbench" title="View models and scores"><strong>KORBench</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>General reasoning</td> <td class="desc">Korean reasoning benchmark evaluating diverse reasoning capabilities.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="79.2"> 79.2% </td> </tr> <tr data-bench="kosimpleqa" data-open-source="1"> <td style="text-align:center" data-sort="307">307</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=kosimpleqa" class="llm-fu-bench-link" data-bench-id="kosimpleqa" title="View models and scores"><strong>KoSimpleQA</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Korean QA</td> <td class="desc">Korean simple question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kanana" data-model-version="Kanana 2" data-provider="kakao"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kanana-2-30b-a3b-mid-2601"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Kanana-2-30B-A3B-Mid-2601</span> </a> </td> <td data-sort="49.7"> 49.7% </td> </tr> <tr data-bench="ksm" data-open-source="1"> <td style="text-align:center" data-sort="308">308</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ksm" class="llm-fu-bench-link" data-bench-id="ksm" title="View models and scores"><strong>KSM</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multilingual math</td> <td class="desc">Korean STEM and math benchmark</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="EXAONE" data-model-version="EXAONE Deep" data-provider="lg"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=exaone-deep-2.4b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">EXAONE Deep 2.4B</span> </a> </td> <td data-sort="60.9"> 60.9% </td> </tr> <tr data-bench="lambada" data-open-source="0"> <td style="text-align:center" data-sort="309">309</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lambada" class="llm-fu-bench-link" data-bench-id="lambada" title="View models and scores"><strong>LAMBADA</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> <a href="https://zenodo.org/record/2630551" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/pdf/1606.06031" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Language modeling</td> <td class="desc">Word prediction requiring broad context understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-3"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-3</span> </a> </td> <td data-sort="86.4"> 86.4% </td> </tr> <tr data-bench="latent-jailbreak" data-open-source="0"> <td style="text-align:center" data-sort="310">310</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=latent-jailbreak" class="llm-fu-bench-link" data-bench-id="latent-jailbreak" title="View models and scores"><strong>LatentJailbreak</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://arxiv.org/pdf/2307.08487.pdf" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/qiuhuachuan/latent-jailbreak" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety / jailbreak</td> <td class="desc">Robustness to latent jailbreak adversarial techniques.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="39">39</td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-3.5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-3.5-turbo"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-3.5-turbo</span> </a> </td> <td data-sort="77.4"> 77.4% </td> </tr> <tr data-bench="lbv1-qa" data-open-source="0"> <td style="text-align:center" data-sort="311">311</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lbv1-qa" class="llm-fu-bench-link" data-bench-id="lbv1-qa" title="View models and scores"><strong>LBV1-QA</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Vision-language</td> <td class="desc">Vision-language QA benchmark v1.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="73.7"> 73.7% </td> </tr> <tr data-bench="lbv2" data-open-source="0"> <td style="text-align:center" data-sort="312">312</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lbv2" class="llm-fu-bench-link" data-bench-id="lbv2" title="View models and scores"><strong>LBV2</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Vision-language</td> <td class="desc">Vision-language benchmark v2.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="65.7"> 65.7% </td> </tr> <tr data-bench="lifebench" data-open-source="0"> <td style="text-align:center" data-sort="313">313</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lifebench" class="llm-fu-bench-link" data-bench-id="lifebench" title="View models and scores"><strong>LIFEBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Instruction following</td> <td class="desc">Length-based instruction-following evaluation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="61.7"> 61.7% </td> </tr> <tr data-bench="lingoqa" data-open-source="1"> <td style="text-align:center" data-sort="314">314</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lingoqa" class="llm-fu-bench-link" data-bench-id="lingoqa" title="View models and scores"><strong>LingoQA</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Driving scene QA</td> <td class="desc">Question answering benchmark for autonomous driving scene understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-27b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-27B</span> </a> </td> <td data-sort="82.0"> 82.0% </td> </tr> <tr data-bench="livebench" data-open-source="0"> <td style="text-align:center" data-sort="315">315</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livebench" class="llm-fu-bench-link" data-bench-id="livebench" title="View models and scores"><strong>LiveBench</strong></a> <span class="score-symbol" title="154 scores" aria-label="154 scores">O</span> </td> <td>General capability</td> <td class="desc">Continually updated capability benchmark across diverse tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="82.4"> 82.4% </td> </tr> <tr data-bench="livecodebench" data-open-source="0"> <td style="text-align:center" data-sort="316">316</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livecodebench" class="llm-fu-bench-link" data-bench-id="livecodebench" title="View models and scores"><strong>LiveCodeBench</strong></a> <span class="score-symbol" title="521 scores" aria-label="521 scores">O</span> <a href="https://livecodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://livecodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">Live coding and execution-based evaluation benchmark (v6 dataset).</td> <td data-sort="3"> <span class="stars" title="Usage: 12/39; GitHub>100: no; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="92.0"> 92.0% </td> </tr> <tr data-bench="livecodebench-ko" data-open-source="1"> <td style="text-align:center" data-sort="317">317</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livecodebench-ko" class="llm-fu-bench-link" data-bench-id="livecodebench-ko" title="View models and scores"><strong>LiveCodeBench-Ko</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Code generation (Korean)</td> <td class="desc">Korean translation of LiveCodeBench.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Thinking-2507</span> </a> </td> <td data-sort="66.3"> 66.3% </td> </tr> <tr data-bench="livecodebench-pro" data-open-source="0"> <td style="text-align:center" data-sort="318">318</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livecodebench-pro" class="llm-fu-bench-link" data-bench-id="livecodebench-pro" title="View models and scores"><strong>LiveCodeBench Pro</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> <a href="https://livecodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://livecodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Competitive coding problems from Codeforces, ICPC, and IOI</td> <td class="desc">LiveCodeBench Pro evaluates competitive programming performance across Codeforces, ICPC, and IOI contests. Elo rating, higher is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="2887"> 2887 </td> </tr> <tr data-bench="livecodebench-pro-25q2-easy" data-open-source="1"> <td style="text-align:center" data-sort="319">319</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livecodebench-pro-25q2-easy" class="llm-fu-bench-link" data-bench-id="livecodebench-pro-25q2-easy" title="View models and scores"><strong>LCB Pro 25Q2 (Easy)</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> <a href="https://livecodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://livecodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">LiveCodeBench Pro 2025 Q2 easy subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nanbeige" data-model-version="Nanbeige4.1" data-provider="boss"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nanbeige4.1-3b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Nanbeige4.1-3B</span> </a> </td> <td data-sort="81.4"> 81.4% </td> </tr> <tr data-bench="livecodebench-pro-25q2-med" data-open-source="1"> <td style="text-align:center" data-sort="320">320</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livecodebench-pro-25q2-med" class="llm-fu-bench-link" data-bench-id="livecodebench-pro-25q2-med" title="View models and scores"><strong>LCB Pro 25Q2 (Med)</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> <a href="https://livecodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://livecodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">LiveCodeBench Pro 2025 Q2 medium subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-OSS" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-oss-120b-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-OSS 120B (High)</span> </a> </td> <td data-sort="35.4"> 35.4% </td> </tr> <tr data-bench="livecodebench-v3" data-open-source="1"> <td style="text-align:center" data-sort="321">321</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livecodebench-v3" class="llm-fu-bench-link" data-bench-id="livecodebench-v3" title="View models and scores"><strong>LiveCodeBench v3</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://livecodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://livecodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">LiveCodeBench v3 snapshot measuring pass rates on streaming coding tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen-3-32b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3 32B</span> </a> </td> <td data-sort="90.2"> 90.2% </td> </tr> <tr data-bench="livecodebench-v5" data-open-source="1"> <td style="text-align:center" data-sort="322">322</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livecodebench-v5" class="llm-fu-bench-link" data-bench-id="livecodebench-v5" title="View models and scores"><strong>LiveCodeBench v5 (2024.10-2025.02)</strong></a> <span class="score-symbol" title="79 scores" aria-label="79 scores">O</span> <a href="https://livecodebench.github.io/leaderboard.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://livecodebench.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation</td> <td class="desc">LiveCodeBench v5 snapshot covering Oct 2024-Feb 2025.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-loop-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Loop-Thinking</span> </a> </td> <td data-sort="86.2"> 86.2% </td> </tr> <tr data-bench="livemcp-101" data-open-source="0"> <td style="text-align:center" data-sort="323">323</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livemcp-101" class="llm-fu-bench-link" data-bench-id="livemcp-101" title="View models and scores"><strong>LiveMCP-101</strong></a> <span class="score-symbol" title="18 scores" aria-label="18 scores">O</span> <a href="https://arxiv.org/pdf/2508.15760" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Agent real-time eval</td> <td class="desc">A novel real-time evaluation framework and benchmark to stress‑test agents on complex, real‑world tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="58.4"> 58.4% </td> </tr> <tr data-bench="livesports-3k" data-open-source="0"> <td style="text-align:center" data-sort="324">324</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=livesports-3k" class="llm-fu-bench-link" data-bench-id="livesports-3k" title="View models and scores"><strong>LiveSports-3K</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Sports video</td> <td class="desc">Live sports video understanding benchmark (3K).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="77.5"> 77.5% </td> </tr> <tr data-bench="lmarena-text" data-open-source="0"> <td style="text-align:center" data-sort="325">325</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lmarena-text" class="llm-fu-bench-link" data-bench-id="lmarena-text" title="View models and scores"><strong>LMArena Text</strong></a> <span class="score-symbol" title="260 scores" aria-label="260 scores">O</span> <a href="https://lmarena.ai/leaderboard/text" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Crowd eval (text)</td> <td class="desc">Chatbot Arena text evaluation (average win rate).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="1455"> 1455 </td> </tr> <tr data-bench="lmarena-vision" data-open-source="0"> <td style="text-align:center" data-sort="326">326</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lmarena-vision" class="llm-fu-bench-link" data-bench-id="lmarena-vision" title="View models and scores"><strong>LMArena Vision</strong></a> <span class="score-symbol" title="79 scores" aria-label="79 scores">O</span> <a href="https://lmarena.ai/leaderboard/vision" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Crowd eval (vision)</td> <td class="desc">Chatbot Arena vision evaluation leaderboard (ELO ratings).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="1242"> 1242 </td> </tr> <tr data-bench="local-agent-bench" data-open-source="1"> <td style="text-align:center" data-sort="327">327</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=local-agent-bench" class="llm-fu-bench-link" data-bench-id="local-agent-bench" title="View models and scores"><strong>Local Agent Bench</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> <a href="https://github.com/MikeVeerman/tool-calling-benchmark" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Tool calling judgment</td> <td class="desc">Tests whether small open-weight models can reliably decide when to call tools and when not to. Agent Score = (Action x 0.4) &#43; (Restraint x 0.3) &#43; (Wrong-Tool-Avoidance x 0.3).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2.5" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2.5-1.2b-instruct"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2.5 1.2B Instruct</span> </a> </td> <td data-sort="88.0"> 88.0% </td> </tr> <tr data-bench="logicvista" data-open-source="0"> <td style="text-align:center" data-sort="328">328</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=logicvista" class="llm-fu-bench-link" data-bench-id="logicvista" title="View models and scores"><strong>LogicVista</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> </td> <td>Visual logical reasoning</td> <td class="desc">Visual logic and pattern reasoning tasks requiring compositional and spatial understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="80.8"> 80.8% </td> </tr> <tr data-bench="logiqa" data-open-source="1"> <td style="text-align:center" data-sort="329">329</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=logiqa" class="llm-fu-bench-link" data-bench-id="logiqa" title="View models and scores"><strong>LogiQA</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://github.com/lgw863/LogiQA-dataset" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Logical reasoning</td> <td class="desc">Reading comprehension with logical reasoning.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="138">138</td> <td class="leader-cell" data-model-family="Pythia" data-model-version="Pythia" data-provider="eleutherai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=pythia-70m"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Pythia 70M</span> </a> </td> <td data-sort="23.5"> 23.5% </td> </tr> <tr data-bench="longbench" data-open-source="0"> <td style="text-align:center" data-sort="330">330</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=longbench" class="llm-fu-bench-link" data-bench-id="longbench" title="View models and scores"><strong>LongBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://github.com/THUDM/LongBench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">Long-context understanding across tasks.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="957">957</td> <td class="leader-cell" data-model-family="Jamba" data-model-version="Jamba Mini" data-provider="ai21"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=jamba-mini-1.6"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Jamba Mini 1.6</span> </a> </td> <td data-sort="32.0"> 32.0% </td> </tr> <tr data-bench="longbench-v2" data-open-source="0"> <td style="text-align:center" data-sort="331">331</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=longbench-v2" class="llm-fu-bench-link" data-bench-id="longbench-v2" title="View models and scores"><strong>longbench-v2</strong></a> <span class="score-symbol" title="61 scores" aria-label="61 scores">O</span> <a href="https://arxiv.org/abs/2412.15204" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">Next-generation LongBench v2 long-context evaluation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="68.2"> 68.2% </td> </tr> <tr data-bench="longfact-concepts" data-open-source="0"> <td style="text-align:center" data-sort="332">332</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=longfact-concepts" class="llm-fu-bench-link" data-bench-id="longfact-concepts" title="View models and scores"><strong>LongFact-Concepts</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Hallucination rate on open-source prompts</td> <td class="desc">Long-context factuality eval focused on conceptual statements; lower is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="0.7"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 0.7%</span> </td> </tr> <tr data-bench="longfact-objects" data-open-source="0"> <td style="text-align:center" data-sort="333">333</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=longfact-objects" class="llm-fu-bench-link" data-bench-id="longfact-objects" title="View models and scores"><strong>LongFact-Objects</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Hallucination rate on open-source prompts</td> <td class="desc">Long-context factuality eval focused on object/entity references; lower is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="0.8"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 0.8%</span> </td> </tr> <tr data-bench="longtext-bench-en" data-open-source="0"> <td style="text-align:center" data-sort="334">334</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=longtext-bench-en" class="llm-fu-bench-link" data-bench-id="longtext-bench-en" title="View models and scores"><strong>LongText-Bench EN</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Text rendering</td> <td class="desc">LongText-Bench English subset score for text rendering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seedream" data-model-version="Seedream" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seedream-4.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seedream 4.5</span> </a> </td> <td data-sort="1.0"> 1.0% </td> </tr> <tr data-bench="longtext-bench-zh" data-open-source="0"> <td style="text-align:center" data-sort="335">335</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=longtext-bench-zh" class="llm-fu-bench-link" data-bench-id="longtext-bench-zh" title="View models and scores"><strong>LongText-Bench ZH</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Text rendering</td> <td class="desc">LongText-Bench Chinese subset score for text rendering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seedream" data-model-version="Seedream" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seedream-4.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seedream 4.5</span> </a> </td> <td data-sort="1.0"> 1.0% </td> </tr> <tr data-bench="longvideobench" data-open-source="1"> <td style="text-align:center" data-sort="336">336</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=longvideobench" class="llm-fu-bench-link" data-bench-id="longvideobench" title="View models and scores"><strong>LongVideoBench</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Long video QA</td> <td class="desc">Long video understanding and QA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5 Thinking</span> </a> </td> <td data-sort="79.8"> 79.8% </td> </tr> <tr data-bench="lpfqa" data-open-source="0"> <td style="text-align:center" data-sort="337">337</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lpfqa" class="llm-fu-bench-link" data-bench-id="lpfqa" title="View models and scores"><strong>LPFQA</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Finance QA</td> <td class="desc">Long-form financial question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5</span> </a> </td> <td data-sort="54.9"> 54.9% </td> </tr> <tr data-bench="lvbench" data-open-source="0"> <td style="text-align:center" data-sort="338">338</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=lvbench" class="llm-fu-bench-link" data-bench-id="lvbench" title="View models and scores"><strong>LVBench</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Video understanding</td> <td class="desc">Long video understanding benchmark (LVBench).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="76.2"> 76.2% </td> </tr> <tr data-bench="m3gia-cn" data-open-source="0"> <td style="text-align:center" data-sort="339">339</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=m3gia-cn" class="llm-fu-bench-link" data-bench-id="m3gia-cn" title="View models and scores"><strong>M3GIA (CN)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Chinese multimodal QA</td> <td class="desc">Chinese-language M3GIA benchmark covering grounded multimodal question answering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.5" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.5-vl-thinking"> <span class="brand"> <img class="logo no-invert" alt="bytedance logo" data-company="bytedance" data-country="China" src="https://furukama.com/logos/bytedance.svg?v=4e7ed8af" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Seed1.5-VL-Thinking</span> </a> </td> <td data-sort="91.2"> 91.2% </td> </tr> <tr data-bench="machiavelli" data-open-source="0"> <td style="text-align:center" data-sort="340">340</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=machiavelli" class="llm-fu-bench-link" data-bench-id="machiavelli" title="View models and scores"><strong>Machiavelli</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Deception / safety</td> <td class="desc">Benchmark for deceptive or manipulative behavior in social interactions.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-4.5-haiku"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Haiku 4.5</span> </a> </td> <td data-sort="52.2"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 52.2%</span> </td> </tr> <tr data-bench="makemesay" data-open-source="0"> <td style="text-align:center" data-sort="341">341</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=makemesay" class="llm-fu-bench-link" data-bench-id="makemesay" title="View models and scores"><strong>MakeMeSay</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Adversarial robustness</td> <td class="desc">Adversarial benchmark testing model robustness against manipulation attempts. Lower is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4.1" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4.1-thinking"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4.1 (Thinking)</span> </a> </td> <td data-sort="0"> </td> </tr> <tr data-bench="mantis" data-open-source="1"> <td style="text-align:center" data-sort="342">342</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mantis" class="llm-fu-bench-link" data-bench-id="mantis" title="View models and scores"><strong>Mantis</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Multimodal reasoning</td> <td class="desc">Multimodal reasoning and instruction following benchmark (Mantis).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="dots" data-model-version="dots.vlm1" data-provider="rednote"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=dots-vlm1"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">dots.vlm1</span> </a> </td> <td data-sort="86.2"> 86.2% </td> </tr> <tr data-bench="marena-hard" data-open-source="1"> <td style="text-align:center" data-sort="343">343</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=marena-hard" class="llm-fu-bench-link" data-bench-id="marena-hard" title="View models and scores"><strong>mArenaHard</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Chat ability (multilingual)</td> <td class="desc">Multilingual variant of Arena-Hard evaluating chat quality across languages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-4b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-4B</span> </a> </td> <td data-sort="70.1"> 70.1% </td> </tr> <tr data-bench="mars-bench" data-open-source="0"> <td style="text-align:center" data-sort="344">344</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mars-bench" class="llm-fu-bench-link" data-bench-id="mars-bench" title="View models and scores"><strong>MARS-Bench</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Instruction following</td> <td class="desc">Instruction-following benchmark with complex tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="87.9"> 87.9% </td> </tr> <tr data-bench="mask" data-open-source="0"> <td style="text-align:center" data-sort="345">345</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mask" class="llm-fu-bench-link" data-bench-id="mask" title="View models and scores"><strong>MASK</strong></a> <span class="score-symbol" title="66 scores" aria-label="66 scores">O</span> <a href="https://scale.com/leaderboard/mask" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/mask" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/html/2503.03750v1" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/mask" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety / red teaming</td> <td class="desc">Model behavior safety assessment via red-teaming scenarios.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4 (t)</span> </a> </td> <td data-sort="95.3"> 95.3% </td> </tr> <tr data-bench="matbench" data-open-source="1"> <td style="text-align:center" data-sort="346">346</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=matbench" class="llm-fu-bench-link" data-bench-id="matbench" title="View models and scores"><strong>MatBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Materials property prediction</td> <td class="desc">Materials property prediction benchmark for scientific AI models.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="Intern-S1" data-provider="internlm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=intern-s1-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Intern-S1-Pro</span> </a> </td> <td data-sort="72.8"> 72.8% </td> </tr> <tr data-bench="math" data-open-source="0"> <td style="text-align:center" data-sort="347">347</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=math" class="llm-fu-bench-link" data-bench-id="math" title="View models and scores"><strong>MATH</strong></a> <span class="score-symbol" title="168 scores" aria-label="168 scores">O</span> <a href="https://github.com/hendrycks/math" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2103.03874" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">Competition-level mathematics across algebra, geometry, number theory, combinatorics.</td> <td data-sort="3"> <span class="stars" title="Usage: 7/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="1185">1185</td> <td class="leader-cell" data-model-family="o3" data-model-version="o3" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o3-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o3 mini</span> </a> </td> <td data-sort="97.9"> 97.9% </td> </tr> <tr data-bench="math-ko" data-open-source="1"> <td style="text-align:center" data-sort="348">348</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=math-ko" class="llm-fu-bench-link" data-bench-id="math-ko" title="View models and scores"><strong>MATH-Ko</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math (Korean)</td> <td class="desc">Korean translation of the MATH competition benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B</span> </a> </td> <td data-sort="58.2"> 58.2% </td> </tr> <tr data-bench="math-lvl5" data-open-source="1"> <td style="text-align:center" data-sort="349">349</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=math-lvl5" class="llm-fu-bench-link" data-bench-id="math-lvl5" title="View models and scores"><strong>MATH Level 5</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://github.com/hendrycks/math" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math (competition)</td> <td class="desc">Level 5 subset of the MATH benchmark emphasizing the hardest competition-style problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-4b-instruct-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-4B-Instruct-2507</span> </a> </td> <td data-sort="73.6"> 73.6% </td> </tr> <tr data-bench="math500" data-open-source="1"> <td style="text-align:center" data-sort="350">350</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=math500" class="llm-fu-bench-link" data-bench-id="math500" title="View models and scores"><strong>MATH500</strong></a> <span class="score-symbol" title="166 scores" aria-label="166 scores">O</span> </td> <td>Math reasoning</td> <td class="desc">500-problem slice of the MATH benchmark for challenging math reasoning.</td> <td data-sort="2"> <span class="stars" title="Usage: 7/39; GitHub>100: no; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Motif" data-model-version="Motif 2" data-provider="motif"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=motif-2-12.7b-reasoning"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Motif-2-12.7B-Reasoning</span> </a> </td> <td data-sort="99.3"> 99.3% </td> </tr> <tr data-bench="math500-es" data-open-source="1"> <td style="text-align:center" data-sort="351">351</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=math500-es" class="llm-fu-bench-link" data-bench-id="math500-es" title="View models and scores"><strong>MATH500 (ES)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Math (multilingual)</td> <td class="desc">Spanish MATH500 benchmark</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="EXAONE" data-model-version="EXAONE 4.0" data-provider="lg"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=exaone-4.0-1.2b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">EXAONE 4.0 1.2B</span> </a> </td> <td data-sort="88.8"> 88.8% </td> </tr> <tr data-bench="matharena-apex" data-open-source="0"> <td style="text-align:center" data-sort="352">352</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=matharena-apex" class="llm-fu-bench-link" data-bench-id="matharena-apex" title="View models and scores"><strong>MathArena Apex</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>Challenging Math Contest problems</td> <td class="desc">Challenging math contest problems from MathArena Apex benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 2.0" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed2.0-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed2.0 Pro</span> </a> </td> <td data-sort="82.1"> 82.1% </td> </tr> <tr data-bench="mathverse" data-open-source="0"> <td style="text-align:center" data-sort="353">353</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mathverse" class="llm-fu-bench-link" data-bench-id="mathverse" title="View models and scores"><strong>MathVerse</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Math reasoning (multimodal)</td> <td class="desc">Visual math reasoning benchmark combining images and text across diverse mathematical tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="82.9"> 82.9% </td> </tr> <tr data-bench="mathverse-mini" data-open-source="1"> <td style="text-align:center" data-sort="354">354</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mathverse-mini" class="llm-fu-bench-link" data-bench-id="mathverse-mini" title="View models and scores"><strong>MathVerse-mini</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Math reasoning (multimodal)</td> <td class="desc">Compact MathVerse split focusing on single-image math puzzles and visual reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-235b-a22b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-235B-A22B Thinking</span> </a> </td> <td data-sort="85.0"> 85.0% </td> </tr> <tr data-bench="mathverse-vision" data-open-source="0"> <td style="text-align:center" data-sort="355">355</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mathverse-vision" class="llm-fu-bench-link" data-bench-id="mathverse-vision" title="View models and scores"><strong>MathVerse-Vision</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>Math reasoning (multimodal)</td> <td class="desc">Multi-image visual mathematical reasoning tasks from the MathVerse ecosystem.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="84.1"> 84.1% </td> </tr> <tr data-bench="mathvision" data-open-source="1"> <td style="text-align:center" data-sort="356">356</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mathvision" class="llm-fu-bench-link" data-bench-id="mathvision" title="View models and scores"><strong>MathVision</strong></a> <span class="score-symbol" title="57 scores" aria-label="57 scores">O</span> <a href="https://mathllm.github.io/mathvision/#leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://mathllm.github.io/mathvision" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://proceedings.neurips.cc/paper_files/paper/2024/file/ad0edc7d5fa1a783f063646968b7315b-Paper-Datasets_and_Benchmarks_Track.pdf" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/mathllm/MATH-V" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Math reasoning (multimodal)</td> <td class="desc">Visual math reasoning benchmark with problems that combine images (charts, diagrams) and text.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="88.6"> 88.6% </td> </tr> <tr data-bench="mathvista" data-open-source="1"> <td style="text-align:center" data-sort="357">357</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mathvista" class="llm-fu-bench-link" data-bench-id="mathvista" title="View models and scores"><strong>MathVista</strong></a> <span class="score-symbol" title="78 scores" aria-label="78 scores">O</span> <a href="https://mathvista.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multimodal math reasoning</td> <td class="desc">Visual math reasoning across diverse tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 4/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5 Thinking</span> </a> </td> <td data-sort="90.1"> 90.1% </td> </tr> <tr data-bench="mathvista-mini" data-open-source="1"> <td style="text-align:center" data-sort="358">358</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mathvista-mini" class="llm-fu-bench-link" data-bench-id="mathvista-mini" title="View models and scores"><strong>MathVista-Mini</strong></a> <span class="score-symbol" title="81 scores" aria-label="81 scores">O</span> </td> <td>Math reasoning (multimodal)</td> <td class="desc">Lightweight subset of MathVista for quick evaluation of visual mathematical reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="90.3"> 90.3% </td> </tr> <tr data-bench="maxife" data-open-source="0"> <td style="text-align:center" data-sort="359">359</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=maxife" class="llm-fu-bench-link" data-bench-id="maxife" title="View models and scores"><strong>MAXIFE</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>Instruction following (multilingual)</td> <td class="desc">Multilingual instruction-following evaluation across English and multilingual original prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="88.4"> 88.4% </td> </tr> <tr data-bench="mbpp" data-open-source="1"> <td style="text-align:center" data-sort="360">360</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mbpp" class="llm-fu-bench-link" data-bench-id="mbpp" title="View models and scores"><strong>MBPP</strong></a> <span class="score-symbol" title="218 scores" aria-label="218 scores">O</span> <a href="https://llm-stats.com/benchmarks" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://huggingface.co/datasets/Muennighoff/mbpp" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2108.07732" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Code generation</td> <td class="desc">Short Python problems with hidden tests.</td> <td data-sort="3"> <span class="stars" title="Usage: 8/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="36312">36312</td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-thinking"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2 Thinking</span> </a> </td> <td data-sort="97.4"> 97.4% </td> </tr> <tr data-bench="mbpp-ko" data-open-source="1"> <td style="text-align:center" data-sort="361">361</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mbpp-ko" class="llm-fu-bench-link" data-bench-id="mbpp-ko" title="View models and scores"><strong>MBPP-Ko</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Code generation (Korean)</td> <td class="desc">Korean translation of MBPP code generation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B</span> </a> </td> <td data-sort="66.8"> 66.8% </td> </tr> <tr data-bench="mbpp-plus" data-open-source="1"> <td style="text-align:center" data-sort="362">362</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mbpp-plus" class="llm-fu-bench-link" data-bench-id="mbpp-plus" title="View models and scores"><strong>MBPP&#43;</strong></a> <span class="score-symbol" title="229 scores" aria-label="229 scores">O</span> </td> <td>Code generation</td> <td class="desc">Extended MBPP with more tests and stricter evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM 4.6</span> </a> </td> <td data-sort="94.2"> 94.2% </td> </tr> <tr data-bench="mcp-atlas" data-open-source="0"> <td style="text-align:center" data-sort="363">363</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mcp-atlas" class="llm-fu-bench-link" data-bench-id="mcp-atlas" title="View models and scores"><strong>MCP-Atlas</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Agent evaluation</td> <td class="desc">Aggregate MCP agent benchmark covering tool-use and planning tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="69.2"> 69.2% </td> </tr> <tr data-bench="mcp-universe" data-open-source="0"> <td style="text-align:center" data-sort="364">364</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mcp-universe" class="llm-fu-bench-link" data-bench-id="mcp-universe" title="View models and scores"><strong>MCP Universe</strong></a> <span class="score-symbol" title="32 scores" aria-label="32 scores">O</span> <a href="https://mcp-universe.github.io/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://mcp-universe.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Agent evaluation</td> <td class="desc">Benchmarks multi-step tool-use agents across diverse task suites with a unified overall success metric.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="50.7"> 50.7% </td> </tr> <tr data-bench="mcpmark" data-open-source="0"> <td style="text-align:center" data-sort="365">365</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mcpmark" class="llm-fu-bench-link" data-bench-id="mcpmark" title="View models and scores"><strong>MCPMark</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> <a href="https://mcpmark.ai/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://mcpmark.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/eval-sys/mcpmark" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Agent tool-use (MCP)</td> <td class="desc">Benchmark for Model Context Protocol (MCP) agent tool-use.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="127">127</td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="57.5"> 57.5% </td> </tr> <tr data-bench="mdolly" data-open-source="1"> <td style="text-align:center" data-sort="366">366</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mdolly" class="llm-fu-bench-link" data-bench-id="mdolly" title="View models and scores"><strong>mDolly</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Instruction following (multilingual)</td> <td class="desc">Multilingual variant of the Dolly instruction-following benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Aya" data-model-version="Aya" data-provider="cohere"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=tiny-aya-global"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Tiny Aya Global</span> </a> </td> <td data-sort="86.9"> 86.9% </td> </tr> <tr data-bench="medxpertqa-mm" data-open-source="0"> <td style="text-align:center" data-sort="367">367</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=medxpertqa-mm" class="llm-fu-bench-link" data-bench-id="medxpertqa-mm" title="View models and scores"><strong>MedXpertQA-MM</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Medical VQA</td> <td class="desc">Multimodal medical expert question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="76.0"> 76.0% </td> </tr> <tr data-bench="metr" data-open-source="0"> <td style="text-align:center" data-sort="368">368</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=metr" class="llm-fu-bench-link" data-bench-id="metr" title="View models and scores"><strong>METR</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> <a href="https://metr.org" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long task benchmark</td> <td class="desc">METR evaluates AI agents on long-horizon coding and agentic tasks, measuring autonomous task completion time.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="4.8"> 4.8% </td> </tr> <tr data-bench="mewc" data-open-source="0"> <td style="text-align:center" data-sort="369">369</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mewc" class="llm-fu-bench-link" data-bench-id="mewc" title="View models and scores"><strong>MEWC</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Web comprehension</td> <td class="desc">Multi-page End-to-end Web Comprehension benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="89.8"> 89.8% </td> </tr> <tr data-bench="mgsm" data-open-source="0"> <td style="text-align:center" data-sort="370">370</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mgsm" class="llm-fu-bench-link" data-bench-id="mgsm" title="View models and scores"><strong>MGSM</strong></a> <span class="score-symbol" title="103 scores" aria-label="103 scores">O</span> </td> <td>Math (multilingual)</td> <td class="desc">Multilingual grade school math word problems.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.1" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4-1-20250805-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.1 (2025-08-05) Thinking</span> </a> </td> <td data-sort="94.4"> 94.4% </td> </tr> <tr data-bench="miabench" data-open-source="0"> <td style="text-align:center" data-sort="371">371</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=miabench" class="llm-fu-bench-link" data-bench-id="miabench" title="View models and scores"><strong>MIABench</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> </td> <td>Multimodal instruction following</td> <td class="desc">Multimodal instruction-following benchmark evaluating accuracy on complex image-text tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="96.0"> 96.0% </td> </tr> <tr data-bench="microvqa" data-open-source="0"> <td style="text-align:center" data-sort="372">372</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=microvqa" class="llm-fu-bench-link" data-bench-id="microvqa" title="View models and scores"><strong>MicroVQA</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Biological microscopy</td> <td class="desc">Visual question answering benchmark for biological microscopy images.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="69.0"> 69.0% </td> </tr> <tr data-bench="mimo-niah-multi-128k" data-open-source="1"> <td style="text-align:center" data-sort="373">373</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mimo-niah-multi-128k" class="llm-fu-bench-link" data-bench-id="mimo-niah-multi-128k" title="View models and scores"><strong>NIAH-Multi 128K</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Long-context QA</td> <td class="desc">Needle-in-a-haystack multi-query benchmark at 128K context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-base"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2 Base</span> </a> </td> <td data-sort="99.5"> 99.5% </td> </tr> <tr data-bench="mimo-niah-multi-32k" data-open-source="1"> <td style="text-align:center" data-sort="374">374</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mimo-niah-multi-32k" class="llm-fu-bench-link" data-bench-id="mimo-niah-multi-32k" title="View models and scores"><strong>NIAH-Multi 32K</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Long-context QA</td> <td class="desc">Needle-in-a-haystack multi-query benchmark at 32K context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-base"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2 Base</span> </a> </td> <td data-sort="99.8"> 99.8% </td> </tr> <tr data-bench="mimo-niah-multi-64k" data-open-source="1"> <td style="text-align:center" data-sort="375">375</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mimo-niah-multi-64k" class="llm-fu-bench-link" data-bench-id="mimo-niah-multi-64k" title="View models and scores"><strong>NIAH-Multi 64K</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Long-context QA</td> <td class="desc">Needle-in-a-haystack multi-query benchmark at 64K context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2-base"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2 Base</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="mindcube" data-open-source="0"> <td style="text-align:center" data-sort="376">376</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mindcube" class="llm-fu-bench-link" data-bench-id="mindcube" title="View models and scores"><strong>MindCube</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Spatial navigation</td> <td class="desc">Spatial navigation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-flash"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Flash</span> </a> </td> <td data-sort="78.3"> 78.3% </td> </tr> <tr data-bench="minerva-math" data-open-source="1"> <td style="text-align:center" data-sort="377">377</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=minerva-math" class="llm-fu-bench-link" data-bench-id="minerva-math" title="View models and scores"><strong>Minerva Math</strong></a> <span class="score-symbol" title="17 scores" aria-label="17 scores">O</span> <a href="https://github.com/allenai/minerva-math" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2206.14858" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/allenai/minerva-math" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>University-level math</td> <td class="desc">Advanced quantitative reasoning set inspired by the Minerva benchmark for STEM problem solving.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen-3-235b-a22b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3 235B A22B Thinking</span> </a> </td> <td data-sort="98.0"> 98.0% </td> </tr> <tr data-bench="minif2f-pass1" data-open-source="1"> <td style="text-align:center" data-sort="378">378</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=minif2f-pass1" class="llm-fu-bench-link" data-bench-id="minif2f-pass1" title="View models and scores"><strong>MiniF2F pass@1</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/deepmind/mathematics_dataset" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math competition</td> <td class="desc">MiniF2F competition benchmark pass@1 accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nvidia-nemotron-3-nano-30b-a3b-bf16"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">NVIDIA-Nemotron-3-Nano-30B-A3B-BF16</span> </a> </td> <td data-sort="50.0"> 50.0% </td> </tr> <tr data-bench="minif2f-pass32" data-open-source="1"> <td style="text-align:center" data-sort="379">379</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=minif2f-pass32" class="llm-fu-bench-link" data-bench-id="minif2f-pass32" title="View models and scores"><strong>MiniF2F pass@32</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/deepmind/mathematics_dataset" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math competition</td> <td class="desc">MiniF2F competition benchmark pass@32 accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nvidia-nemotron-3-nano-30b-a3b-bf16"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">NVIDIA-Nemotron-3-Nano-30B-A3B-BF16</span> </a> </td> <td data-sort="79.9"> 79.9% </td> </tr> <tr data-bench="minif2f-test" data-open-source="1"> <td style="text-align:center" data-sort="380">380</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=minif2f-test" class="llm-fu-bench-link" data-bench-id="minif2f-test" title="View models and scores"><strong>MiniF2F (Test)</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://github.com/deepmind/mathematics_dataset" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Math competition</td> <td class="desc">MiniF2F competition benchmark (test split).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LongCat" data-model-version="LongCat-Flash" data-provider="meituan"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=longcat-flash-thinking"> <span class="brand"> <img class="logo no-invert" alt="meituan logo" data-company="meituan" data-country="China" src="https://furukama.com/logos/meituan.svg?v=45908c84" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">LongCat-Flash-Thinking</span> </a> </td> <td data-sort="81.6"> 81.6% </td> </tr> <tr data-bench="mixeval" data-open-source="0"> <td style="text-align:center" data-sort="381">381</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mixeval" class="llm-fu-bench-link" data-bench-id="mixeval" title="View models and scores"><strong>MixEval</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Multi-task reasoning</td> <td class="desc">Mixed-subject benchmark covering knowledge and reasoning tasks across domains.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="o1" data-model-version="o1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o1-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o1 Mini</span> </a> </td> <td data-sort="82.9"> 82.9% </td> </tr> <tr data-bench="mixeval-hard" data-open-source="1"> <td style="text-align:center" data-sort="382">382</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mixeval-hard" class="llm-fu-bench-link" data-bench-id="mixeval-hard" title="View models and scores"><strong>MixEval Hard</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multi-task reasoning (hard)</td> <td class="desc">Hard subset of MixEval covering diverse reasoning tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-4b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-4B</span> </a> </td> <td data-sort="31.6"> 31.6% </td> </tr> <tr data-bench="mlvu" data-open-source="1"> <td style="text-align:center" data-sort="383">383</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mlvu" class="llm-fu-bench-link" data-bench-id="mlvu" title="View models and scores"><strong>MLVU</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> </td> <td>Large video understanding</td> <td class="desc">MLVU: Large-scale multi-task benchmark for video understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-122b-a10b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-122B-A10B</span> </a> </td> <td data-sort="87.3"> 87.3% </td> </tr> <tr data-bench="mm-browsecomp" data-open-source="0"> <td style="text-align:center" data-sort="384">384</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mm-browsecomp" class="llm-fu-bench-link" data-bench-id="mm-browsecomp" title="View models and scores"><strong>MM-BrowseComp</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Multimodal browsing</td> <td class="desc">Multimodal browsing comprehension benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="46.3"> 46.3% </td> </tr> <tr data-bench="mm-ifeval" data-open-source="1"> <td style="text-align:center" data-sort="385">385</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mm-ifeval" class="llm-fu-bench-link" data-bench-id="mm-ifeval" title="View models and scores"><strong>MM-IFEval</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> <a href="https://arxiv.org/abs/2502.04688" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Multimodal instruction following</td> <td class="desc">Instruction-following benchmark assessing multimodal obedience to complex prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2.5-VL" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2.5-vl-1.6b"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2.5-VL-1.6B</span> </a> </td> <td data-sort="52.3"> 52.3% </td> </tr> <tr data-bench="mm-mt-bench" data-open-source="1"> <td style="text-align:center" data-sort="386">386</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mm-mt-bench" class="llm-fu-bench-link" data-bench-id="mm-mt-bench" title="View models and scores"><strong>MM-MT-Bench</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> </td> <td>Multimodal instruction following</td> <td class="desc">Multi-turn multimodal instruction following benchmark evaluating dialogue quality and helpfulness.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-235b-a22b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-235B-A22B Thinking</span> </a> </td> <td data-sort="8.5"> 8.5% </td> </tr> <tr data-bench="mmbench-v1.1-cn" data-open-source="0"> <td style="text-align:center" data-sort="387">387</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmbench-v1.1-cn" class="llm-fu-bench-link" data-bench-id="mmbench-v1.1-cn" title="View models and scores"><strong>MMBench v1.1 (CN)</strong></a> <span class="score-symbol" title="28 scores" aria-label="28 scores">O</span> <a href="https://mmbench.opencompass.org.cn/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2307.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Multimodal understanding (Chinese)</td> <td class="desc">MMBench v1.1 Chinese subset for evaluating multimodal LLMs.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="91.3"> 91.3% </td> </tr> <tr data-bench="mmbench-v1.1-en" data-open-source="0"> <td style="text-align:center" data-sort="388">388</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmbench-v1.1-en" class="llm-fu-bench-link" data-bench-id="mmbench-v1.1-en" title="View models and scores"><strong>MMBench v1.1 (EN)</strong></a> <span class="score-symbol" title="40 scores" aria-label="40 scores">O</span> <a href="https://mmbench.opencompass.org.cn/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2307.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Multimodal understanding (English)</td> <td class="desc">MMBench v1.1 English subset for evaluating multimodal LLMs.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="93.3"> 93.3% </td> </tr> <tr data-bench="mmbench-v1.1-en-dev" data-open-source="1"> <td style="text-align:center" data-sort="389">389</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmbench-v1.1-en-dev" class="llm-fu-bench-link" data-bench-id="mmbench-v1.1-en-dev" title="View models and scores"><strong>MMBench v1.1 (EN dev)</strong></a> <span class="score-symbol" title="54 scores" aria-label="54 scores">O</span> <a href="https://mmbench.opencompass.org.cn/leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://mmbench.opencompass.org.cn/home" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2307.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/open-compass/MMBench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>General VQA</td> <td class="desc">English dev split of MMBench v1.1 measuring multimodal question answering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5</span> </a> </td> <td data-sort="94.2"> 94.2% </td> </tr> <tr data-bench="mme-cc" data-open-source="0"> <td style="text-align:center" data-sort="390">390</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mme-cc" class="llm-fu-bench-link" data-bench-id="mme-cc" title="View models and scores"><strong>MME-CC</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multimodal evaluation</td> <td class="desc">MME-CC multimodal evaluation suite.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="56.9"> 56.9% </td> </tr> <tr data-bench="mme-elo" data-open-source="1"> <td style="text-align:center" data-sort="391">391</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mme-elo" class="llm-fu-bench-link" data-bench-id="mme-elo" title="View models and scores"><strong>MME Elo</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multimodal perception</td> <td class="desc">Elo-style scoring for the MME multimodal evaluation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="InternVL 3" data-provider="opengvlab"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=internvl3-2b"> <span class="brand"> <img class="logo no-invert" alt="opengvlab logo" data-company="opengvlab" data-country="China" src="https://furukama.com/logos/opengvlab.png?v=e14fd644" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">InternVL3-2B</span> </a> </td> <td data-sort="2186.4"> 2186.4% </td> </tr> <tr data-bench="mme-realworld-cn" data-open-source="0"> <td style="text-align:center" data-sort="392">392</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mme-realworld-cn" class="llm-fu-bench-link" data-bench-id="mme-realworld-cn" title="View models and scores"><strong>MME-RealWorld (cn)</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Real-world perception (CN)</td> <td class="desc">MME-RealWorld Chinese split.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4o" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4o"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4o</span> </a> </td> <td data-sort="58.5"> 58.5% </td> </tr> <tr data-bench="mme-realworld-en" data-open-source="1"> <td style="text-align:center" data-sort="393">393</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mme-realworld-en" class="llm-fu-bench-link" data-bench-id="mme-realworld-en" title="View models and scores"><strong>MME-RealWorld (en)</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Real-world perception (EN)</td> <td class="desc">MME-RealWorld English split.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiMo" data-model-version="MiMo VL" data-provider="xiaomi"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mimo-vl-7b-rl"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiMo-VL 7B-RL</span> </a> </td> <td data-sort="59.1"> 59.1% </td> </tr> <tr data-bench="mmiu" data-open-source="0"> <td style="text-align:center" data-sort="394">394</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmiu" class="llm-fu-bench-link" data-bench-id="mmiu" title="View models and scores"><strong>MMIU</strong></a> <span class="score-symbol" title="19 scores" aria-label="19 scores">O</span> </td> <td>Multi-image understanding</td> <td class="desc">Multi-image understanding benchmark evaluating cross-image reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="72.1"> 72.1% </td> </tr> <tr data-bench="mmlb-niah-128k" data-open-source="0"> <td style="text-align:center" data-sort="395">395</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlb-niah-128k" class="llm-fu-bench-link" data-bench-id="mmlb-niah-128k" title="View models and scores"><strong>MMLB-NIAH (128k)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Multimodal long-context</td> <td class="desc">MMLB-NIAH 128k long-context multimodal benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="72.2"> 72.2% </td> </tr> <tr data-bench="mmlb-vrag-128k" data-open-source="0"> <td style="text-align:center" data-sort="396">396</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlb-vrag-128k" class="llm-fu-bench-link" data-bench-id="mmlb-vrag-128k" title="View models and scores"><strong>MMLB-VRAG (128k)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Multimodal long-context</td> <td class="desc">MMLB-VRAG 128k long-context multimodal benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="88.9"> 88.9% </td> </tr> <tr data-bench="mmlongbench-128k" data-open-source="1"> <td style="text-align:center" data-sort="397">397</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlongbench-128k" class="llm-fu-bench-link" data-bench-id="mmlongbench-128k" title="View models and scores"><strong>MMLongBench-128K</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Long-context multimodal</td> <td class="desc">128K-context variant of MMLongBench evaluating multimodal long-context understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.6V</span> </a> </td> <td data-sort="64.1"> 64.1% </td> </tr> <tr data-bench="mmlongbench-doc" data-open-source="0"> <td style="text-align:center" data-sort="398">398</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlongbench-doc" class="llm-fu-bench-link" data-bench-id="mmlongbench-doc" title="View models and scores"><strong>MMLongBench-Doc</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Long-context multimodal documents</td> <td class="desc">Evaluates long-context document understanding with mixed text, tables, and figures across multiple pages.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="61.9"> 61.9% </td> </tr> <tr data-bench="mmlu" data-open-source="0"> <td style="text-align:center" data-sort="399">399</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu" class="llm-fu-bench-link" data-bench-id="mmlu" title="View models and scores"><strong>MMLU</strong></a> <span class="score-symbol" title="393 scores" aria-label="393 scores">O</span> <a href="https://www.kaggle.com/benchmarks/open-benchmarks/mmlu" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/hendrycks/test" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2009.03300" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/hendrycks/test" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multi-domain knowledge</td> <td class="desc">57 tasks spanning STEM, humanities, social sciences; broad knowledge and reasoning.</td> <td data-sort="5"> <span class="stars" title="Usage: 18/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★★★ </span> </td> <td data-sort="1488">1488</td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="93.8"> 93.8% </td> </tr> <tr data-bench="mmlu-arabic" data-open-source="1"> <td style="text-align:center" data-sort="400">400</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-arabic" class="llm-fu-bench-link" data-bench-id="mmlu-arabic" title="View models and scores"><strong>MMLU Arabic</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Arabic knowledge and reasoning</td> <td class="desc">Arabic-language variant of MMLU evaluating knowledge and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen-2.5-72b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen 2.5 72B</span> </a> </td> <td data-sort="74.1"> 74.1% </td> </tr> <tr data-bench="mmlu-cloze" data-open-source="1"> <td style="text-align:center" data-sort="401">401</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-cloze" class="llm-fu-bench-link" data-bench-id="mmlu-cloze" title="View models and scores"><strong>MMLU (cloze)</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/hendrycks/test" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/hendrycks/test" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multi-domain knowledge (cloze)</td> <td class="desc">Cloze-form MMLU evaluation variant.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="SmolLM" data-model-version="SmolLM2" data-provider="huggingface"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=smollm2-135m-base"> <span class="brand"> <img class="logo no-invert" alt="huggingface logo" data-company="huggingface" data-country="United States" src="https://furukama.com/logos/huggingface.svg?v=5cc8e86e" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">SmolLM2 135M Base</span> </a> </td> <td data-sort="31.5"> 31.5% </td> </tr> <tr data-bench="mmlu-fulltext" data-open-source="1"> <td style="text-align:center" data-sort="402">402</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-fulltext" class="llm-fu-bench-link" data-bench-id="mmlu-fulltext" title="View models and scores"><strong>Full Text MMLU</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Multi-domain knowledge (long-form)</td> <td class="desc">Full-context MMLU variant evaluating reasoning over long passages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b-instruct"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B Instruct</span> </a> </td> <td data-sort="83.0"> 83.0% </td> </tr> <tr data-bench="mmlu-pro" data-open-source="0"> <td style="text-align:center" data-sort="403">403</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-pro" class="llm-fu-bench-link" data-bench-id="mmlu-pro" title="View models and scores"><strong>MMLU-Pro</strong></a> <span class="score-symbol" title="405 scores" aria-label="405 scores">O</span> <a href="https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://github.com/TIGER-AI-Lab/MMLU-Pro" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2406.01574" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/TIGER-AI-Lab/MMLU-Pro" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multi-domain knowledge</td> <td class="desc">Harder successor to MMLU with more challenging questions.</td> <td data-sort="4"> <span class="stars" title="Usage: 12/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★★<span class="empty">★</span> </span> </td> <td data-sort="286">286</td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="90.1"> 90.1% </td> </tr> <tr data-bench="mmlu-pro-mcf" data-open-source="1"> <td style="text-align:center" data-sort="404">404</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-pro-mcf" class="llm-fu-bench-link" data-bench-id="mmlu-pro-mcf" title="View models and scores"><strong>MMLU Pro MCF</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Multi-domain knowledge (few-shot)</td> <td class="desc">MMLU-Pro common format (MCF) few-shot evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-4b-base"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-4B-Base</span> </a> </td> <td data-sort="41.1"> 41.1% </td> </tr> <tr data-bench="mmlu-prox" data-open-source="0"> <td style="text-align:center" data-sort="405">405</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-prox" class="llm-fu-bench-link" data-bench-id="mmlu-prox" title="View models and scores"><strong>MMLU-ProX</strong></a> <span class="score-symbol" title="32 scores" aria-label="32 scores">O</span> </td> <td>Multi-domain knowledge</td> <td class="desc">Cross-lingual and robust variant of MMLU-Pro.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="87.7"> 87.7% </td> </tr> <tr data-bench="mmlu-redux" data-open-source="0"> <td style="text-align:center" data-sort="406">406</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-redux" class="llm-fu-bench-link" data-bench-id="mmlu-redux" title="View models and scores"><strong>MMLU-Redux</strong></a> <span class="score-symbol" title="81 scores" aria-label="81 scores">O</span> <a href="https://huggingface.co/spaces/allenai/ZeroEval" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Multi-domain knowledge</td> <td class="desc">Updated MMLU-style evaluation with revised questions and scoring.</td> <td data-sort="1"> <span class="stars" title="Usage: 4/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="95.9"> 95.9% </td> </tr> <tr data-bench="mmlu-stem" data-open-source="1"> <td style="text-align:center" data-sort="407">407</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmlu-stem" class="llm-fu-bench-link" data-bench-id="mmlu-stem" title="View models and scores"><strong>MMLU-STEM</strong></a> <span class="score-symbol" title="30 scores" aria-label="30 scores">O</span> <a href="https://github.com/hendrycks/test" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>STEM knowledge</td> <td class="desc">STEM subset of MMLU.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="1488">1488</td> <td class="leader-cell" data-model-family="Falcon" data-model-version="Falcon H1" data-provider="tii"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=falcon-h1-34b-instruct"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Falcon-H1-34B-Instruct</span> </a> </td> <td data-sort="83.6"> 83.6% </td> </tr> <tr data-bench="mmmb" data-open-source="1"> <td style="text-align:center" data-sort="408">408</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmmb" class="llm-fu-bench-link" data-bench-id="mmmb" title="View models and scores"><strong>MMMB</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Multilingual MMBench</td> <td class="desc">Multilingual Multimodal Benchmark (MMMB) average score.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2.5-VL" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2.5-vl-1.6b"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2.5-VL-1.6B</span> </a> </td> <td data-sort="77.0"> 77.0% </td> </tr> <tr data-bench="mmmlu" data-open-source="0"> <td style="text-align:center" data-sort="409">409</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmmlu" class="llm-fu-bench-link" data-bench-id="mmmlu" title="View models and scores"><strong>MMMLU</strong></a> <span class="score-symbol" title="71 scores" aria-label="71 scores">O</span> <a href="https://huggingface.co/LiquidAI/LFM2-350M" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multi-domain knowledge (multilingual)</td> <td class="desc">Massively multilingual MMLU-style evaluation across many languages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="92.6"> 92.6% </td> </tr> <tr data-bench="mmmlu-es" data-open-source="1"> <td style="text-align:center" data-sort="410">410</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmmlu-es" class="llm-fu-bench-link" data-bench-id="mmmlu-es" title="View models and scores"><strong>MMMLU (ES)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multilingual knowledge</td> <td class="desc">Spanish MMMLU benchmark</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="SmolLM" data-model-version="SmolLM3" data-provider="huggingface"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=smollm-3-3b"> <span class="brand"> <img class="logo no-invert" alt="huggingface logo" data-company="huggingface" data-country="United States" src="https://furukama.com/logos/huggingface.svg?v=5cc8e86e" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">SmolLM 3 3B</span> </a> </td> <td data-sort="64.7"> 64.7% </td> </tr> <tr data-bench="mmmu" data-open-source="0"> <td style="text-align:center" data-sort="411">411</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmmu" class="llm-fu-bench-link" data-bench-id="mmmu" title="View models and scores"><strong>MMMU</strong></a> <span class="score-symbol" title="186 scores" aria-label="186 scores">O</span> <a href="https://mmmu-benchmark.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multimodal understanding</td> <td class="desc">Multi-discipline multimodal understanding benchmark.</td> <td data-sort="2"> <span class="stars" title="Usage: 11/39; GitHub>100: no; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="87.2"> 87.2% </td> </tr> <tr data-bench="mmmu-pro" data-open-source="0"> <td style="text-align:center" data-sort="412">412</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmmu-pro" class="llm-fu-bench-link" data-bench-id="mmmu-pro" title="View models and scores"><strong>MMMU PRO</strong></a> <span class="score-symbol" title="82 scores" aria-label="82 scores">O</span> <a href="https://mmmu-benchmark.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multimodal understanding (hard)</td> <td class="desc">Professional/advanced subset of MMMU for multimodal reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-deep-think"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Deep Think</span> </a> </td> <td data-sort="81.5"> 81.5% </td> </tr> <tr data-bench="mmmu-pro-vision" data-open-source="0"> <td style="text-align:center" data-sort="413">413</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmmu-pro-vision" class="llm-fu-bench-link" data-bench-id="mmmu-pro-vision" title="View models and scores"><strong>MMMU-Pro (vision)</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://mmmu-benchmark.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multimodal understanding (vision)</td> <td class="desc">MMMU-Pro vision-only setting.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 3.7" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-3.7-sonnet"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude 3.7 Sonnet</span> </a> </td> <td data-sort="45.8"> 45.8% </td> </tr> <tr data-bench="mmmu-pro-with-tools" data-open-source="0"> <td style="text-align:center" data-sort="414">414</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmmu-pro-with-tools" class="llm-fu-bench-link" data-bench-id="mmmu-pro-with-tools" title="View models and scores"><strong>MMMU Pro (with tools)</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://mmmu-benchmark.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Multimodal understanding (with tools)</td> <td class="desc">MMMU-Pro benchmark evaluated with tool access.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="80.4"> 80.4% </td> </tr> <tr data-bench="mmsibench-circular" data-open-source="0"> <td style="text-align:center" data-sort="415">415</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmsibench-circular" class="llm-fu-bench-link" data-bench-id="mmsibench-circular" title="View models and scores"><strong>MMSIBench (circular)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Spatial understanding</td> <td class="desc">MMSIBench circular subset for spatial reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="25.4"> 25.4% </td> </tr> <tr data-bench="mmstar" data-open-source="1"> <td style="text-align:center" data-sort="416">416</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmstar" class="llm-fu-bench-link" data-bench-id="mmstar" title="View models and scores"><strong>MMStar</strong></a> <span class="score-symbol" title="73 scores" aria-label="73 scores">O</span> <a href="https://mmstar-benchmark.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2403.20330" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/MMStar-Benchmark/MMStar" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multimodal reasoning</td> <td class="desc">Broad evaluation of multimodal LLMs across diverse tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="83.8"> 83.8% </td> </tr> <tr data-bench="mmvet" data-open-source="1"> <td style="text-align:center" data-sort="417">417</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmvet" class="llm-fu-bench-link" data-bench-id="mmvet" title="View models and scores"><strong>MMVet</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://mm-vet.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2308.02490" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/yuweihao/MM-Vet" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multimodal evaluation</td> <td class="desc">Comprehensive evaluation suite for assessing multimodal LLM capabilities.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="R" data-model-version="R" data-provider="tencent"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=r-4b-base"> <span class="brand"> <img class="logo" alt="tencent logo" data-company="tencent" data-country="China" src="https://furukama.com/logos/tencent.svg?v=ce01b16a" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">R-4B-Base</span> </a> </td> <td data-sort="85.9"> 85.9% </td> </tr> <tr data-bench="mmvp" data-open-source="0"> <td style="text-align:center" data-sort="418">418</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmvp" class="llm-fu-bench-link" data-bench-id="mmvp" title="View models and scores"><strong>MMVP</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> <a href="https://mmvp-benchmark.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2308.16154" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/Metaverse-AI-Lab-THU/MMVP-Dataset" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multimodal video perception</td> <td class="desc">Benchmark for multimodal video understanding and perception.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="91.6"> 91.6% </td> </tr> <tr data-bench="mmvu" data-open-source="0"> <td style="text-align:center" data-sort="419">419</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mmvu" class="llm-fu-bench-link" data-bench-id="mmvu" title="View models and scores"><strong>MMVU</strong></a> <span class="score-symbol" title="23 scores" aria-label="23 scores">O</span> </td> <td>Video understanding</td> <td class="desc">Multimodal video understanding benchmark (MMVU).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-thinking-xhigh"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Thinking XHigh</span> </a> </td> <td data-sort="80.8"> 80.8% </td> </tr> <tr data-bench="mol-instructions" data-open-source="1"> <td style="text-align:center" data-sort="420">420</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mol-instructions" class="llm-fu-bench-link" data-bench-id="mol-instructions" title="View models and scores"><strong>Mol-Instructions</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Bio-molecular instruction following</td> <td class="desc">Instruction-following benchmark for bio-molecular understanding and generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="Intern-S1" data-provider="internlm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=intern-s1-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Intern-S1-Pro</span> </a> </td> <td data-sort="48.8"> 48.8% </td> </tr> <tr data-bench="motionbench" data-open-source="0"> <td style="text-align:center" data-sort="421">421</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=motionbench" class="llm-fu-bench-link" data-bench-id="motionbench" title="View models and scores"><strong>MotionBench</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Video motion understanding</td> <td class="desc">Video motion and temporal reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="70.6"> 70.6% </td> </tr> <tr data-bench="mrcr-128k" data-open-source="0"> <td style="text-align:center" data-sort="422">422</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-128k" class="llm-fu-bench-link" data-bench-id="mrcr-128k" title="View models and scores"><strong>OpenAI-MRCR (128k)</strong></a> <span class="score-symbol" title="39 scores" aria-label="39 scores">O</span> </td> <td>Long-context reasoning</td> <td class="desc">OpenAI Multi-Round Chain Reasoning benchmark with 128k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="89.7"> 89.7% </td> </tr> <tr data-bench="mrcr-128k-2n" data-open-source="1"> <td style="text-align:center" data-sort="423">423</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-128k-2n" class="llm-fu-bench-link" data-bench-id="mrcr-128k-2n" title="View models and scores"><strong>MRCR 128K-2N</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Multi-Round Coreference Resolution benchmark at 128k context with 2 needles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ministral" data-model-version="Ministral 3" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ministral-3-r-8b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Ministral-3-R 8B</span> </a> </td> <td data-sort="50.3"> 50.3% </td> </tr> <tr data-bench="mrcr-128k-4n" data-open-source="1"> <td style="text-align:center" data-sort="424">424</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-128k-4n" class="llm-fu-bench-link" data-bench-id="mrcr-128k-4n" title="View models and scores"><strong>MRCR 128K-4N</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Multi-Round Coreference Resolution benchmark at 128k context with 4 needles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ministral" data-model-version="Ministral 3" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ministral-3-r-8b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Ministral-3-R 8B</span> </a> </td> <td data-sort="22.7"> 22.7% </td> </tr> <tr data-bench="mrcr-128k-8n" data-open-source="0"> <td style="text-align:center" data-sort="425">425</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-128k-8n" class="llm-fu-bench-link" data-bench-id="mrcr-128k-8n" title="View models and scores"><strong>MRCR 128K-8N</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Long-context reasoning</td> <td class="desc">Multi-Round Coreference Resolution benchmark at 128k context with 8 needles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="84.9"> 84.9% </td> </tr> <tr data-bench="mrcr-1m" data-open-source="0"> <td style="text-align:center" data-sort="426">426</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-1m" class="llm-fu-bench-link" data-bench-id="mrcr-1m" title="View models and scores"><strong>OpenAI-MRCR (1M)</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Long-context reasoning</td> <td class="desc">OpenAI Multi-Round Chain Reasoning benchmark with 1M context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="58.8"> 58.8% </td> </tr> <tr data-bench="mrcr-64k-2n" data-open-source="1"> <td style="text-align:center" data-sort="427">427</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-64k-2n" class="llm-fu-bench-link" data-bench-id="mrcr-64k-2n" title="View models and scores"><strong>MRCR 64K-2N</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Multi-Round Coreference Resolution benchmark at 64k context with 2 needles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ministral" data-model-version="Ministral 3" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ministral-3-r-8b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Ministral-3-R 8B</span> </a> </td> <td data-sort="44.0"> 44.0% </td> </tr> <tr data-bench="mrcr-64k-4n" data-open-source="1"> <td style="text-align:center" data-sort="428">428</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-64k-4n" class="llm-fu-bench-link" data-bench-id="mrcr-64k-4n" title="View models and scores"><strong>MRCR 64K-4N</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Multi-Round Coreference Resolution benchmark at 64k context with 4 needles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ministral" data-model-version="Ministral 3" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ministral-3-r-8b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Ministral-3-R 8B</span> </a> </td> <td data-sort="35.8"> 35.8% </td> </tr> <tr data-bench="mrcr-64k-8n" data-open-source="1"> <td style="text-align:center" data-sort="429">429</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-64k-8n" class="llm-fu-bench-link" data-bench-id="mrcr-64k-8n" title="View models and scores"><strong>MRCR 64K-8N</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context reasoning</td> <td class="desc">Multi-Round Coreference Resolution benchmark at 64k context with 8 needles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-8b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-8B</span> </a> </td> <td data-sort="17.8"> 17.8% </td> </tr> <tr data-bench="mrcr-v2" data-open-source="0"> <td style="text-align:center" data-sort="430">430</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mrcr-v2" class="llm-fu-bench-link" data-bench-id="mrcr-v2" title="View models and scores"><strong>MRCR v2</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Multimodal reasoning</td> <td class="desc">Multi-round multimodal chain-of-reasoning evaluation (v2).</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 High</span> </a> </td> <td data-sort="89.4"> 89.4% </td> </tr> <tr data-bench="msearth-mcq" data-open-source="0"> <td style="text-align:center" data-sort="431">431</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=msearth-mcq" class="llm-fu-bench-link" data-bench-id="msearth-mcq" title="View models and scores"><strong>MSEarth-MCQ</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Earth science</td> <td class="desc">Earth science multiple-choice question benchmark for scientific AI models.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="65.8"> 65.8% </td> </tr> <tr data-bench="mt-bench" data-open-source="1"> <td style="text-align:center" data-sort="432">432</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mt-bench" class="llm-fu-bench-link" data-bench-id="mt-bench" title="View models and scores"><strong>MT-Bench</strong></a> <span class="score-symbol" title="46 scores" aria-label="46 scores">O</span> <a href="https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2306.05685v4" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Chat ability</td> <td class="desc">Multi-turn chat evaluation via GPT-4 grading.</td> <td data-sort="2"> <span class="stars" title="Usage: 1/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="39074">39074</td> <td class="leader-cell" data-model-family="Apriel" data-model-version="Apriel Nemotron" data-provider="servicenow"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=apriel-nemotron-15b-thinker"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Apriel Nemotron 15B Thinker</span> </a> </td> <td data-sort="85.7"> 85.7% </td> </tr> <tr data-bench="mtob-full-book" data-open-source="1"> <td style="text-align:center" data-sort="433">433</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mtob-full-book" class="llm-fu-bench-link" data-bench-id="mtob-full-book" title="View models and scores"><strong>MTOB (full book)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Long-form reasoning</td> <td class="desc">Long-context book understanding benchmark (full-book setting).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 4" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-4-maverick"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 4 Maverick</span> </a> </td> <td data-sort="50.8"> 50.8% </td> </tr> <tr data-bench="mtob-half-book" data-open-source="1"> <td style="text-align:center" data-sort="434">434</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mtob-half-book" class="llm-fu-bench-link" data-bench-id="mtob-half-book" title="View models and scores"><strong>MTOB (half book)</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Long-form reasoning</td> <td class="desc">Long-context book understanding benchmark (half-book setting).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 4" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-4-maverick"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 4 Maverick</span> </a> </td> <td data-sort="54.0"> 54.0% </td> </tr> <tr data-bench="muirbench" data-open-source="0"> <td style="text-align:center" data-sort="435">435</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=muirbench" class="llm-fu-bench-link" data-bench-id="muirbench" title="View models and scores"><strong>MUIRBENCH</strong></a> <span class="score-symbol" title="51 scores" aria-label="51 scores">O</span> </td> <td>Multimodal robustness</td> <td class="desc">Evaluates multimodal understanding robustness and reliability.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="86.1"> 86.1% </td> </tr> <tr data-bench="multi-if" data-open-source="1"> <td style="text-align:center" data-sort="436">436</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multi-if" class="llm-fu-bench-link" data-bench-id="multi-if" title="View models and scores"><strong>Multi-IF</strong></a> <span class="score-symbol" title="56 scores" aria-label="56 scores">O</span> </td> <td>Instruction following (multi-task)</td> <td class="desc">Composite instruction-following evaluation across multiple tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B</span> </a> </td> <td data-sort="81.0"> 81.0% </td> </tr> <tr data-bench="multi-ifeval" data-open-source="1"> <td style="text-align:center" data-sort="437">437</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multi-ifeval" class="llm-fu-bench-link" data-bench-id="multi-ifeval" title="View models and scores"><strong>Multi-IFEval</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Instruction following (multi-task)</td> <td class="desc">Multi-task variant of instruction-following evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B</span> </a> </td> <td data-sort="88.7"> 88.7% </td> </tr> <tr data-bench="multi-swe-bench" data-open-source="1"> <td style="text-align:center" data-sort="438">438</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multi-swe-bench" class="llm-fu-bench-link" data-bench-id="multi-swe-bench" title="View models and scores"><strong>Multi-SWE-Bench</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> <a href="https://multi-swe-bench.github.io" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code repair (multi-repo)</td> <td class="desc">Multi-repository SWE-Bench variant.</td> <td data-sort="2"> <span class="stars" title="Usage: 1/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="246">246</td> <td class="leader-cell" data-model-family="MiniMax" data-model-version="MiniMax M2.5" data-provider="minimax"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minimax-m2.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiniMax M2.5</span> </a> </td> <td data-sort="51.3"> 51.3% </td> </tr> <tr data-bench="multichallenge" data-open-source="0"> <td style="text-align:center" data-sort="439">439</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multichallenge" class="llm-fu-bench-link" data-bench-id="multichallenge" title="View models and scores"><strong>MultiChallenge</strong></a> <span class="score-symbol" title="75 scores" aria-label="75 scores">O</span> </td> <td>Instruction following</td> <td class="desc">Multi-domain instruction-following benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="69.6"> 69.6% </td> </tr> <tr data-bench="multiimg-qa-avg" data-open-source="0"> <td style="text-align:center" data-sort="440">440</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multiimg-qa-avg" class="llm-fu-bench-link" data-bench-id="multiimg-qa-avg" title="View models and scores"><strong>Multi-Image QA Average</strong></a> <span class="score-symbol" title="19 scores" aria-label="19 scores">O</span> </td> <td>Multi-image QA (aggregate)</td> <td class="desc">Aggregate score over multi-image visual question answering tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="81.9"> 81.9% </td> </tr> <tr data-bench="multilingual-mmbench" data-open-source="1"> <td style="text-align:center" data-sort="441">441</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multilingual-mmbench" class="llm-fu-bench-link" data-bench-id="multilingual-mmbench" title="View models and scores"><strong>Multilingual MMBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Multilingual vision benchmark</td> <td class="desc">Multilingual MMBench average score across languages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2.5-VL" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2.5-vl-1.6b"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2.5-VL-1.6B</span> </a> </td> <td data-sort="65.9"> 65.9% </td> </tr> <tr data-bench="multilingual-mmlu" data-open-source="0"> <td style="text-align:center" data-sort="442">442</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multilingual-mmlu" class="llm-fu-bench-link" data-bench-id="multilingual-mmlu" title="View models and scores"><strong>Multilingual MMLU</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>Multi-domain knowledge (multilingual)</td> <td class="desc">Multilingual variant of MMLU across many languages.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4.1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4.1"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4.1</span> </a> </td> <td data-sort="87.3"> 87.3% </td> </tr> <tr data-bench="multipl-e" data-open-source="0"> <td style="text-align:center" data-sort="443">443</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multipl-e" class="llm-fu-bench-link" data-bench-id="multipl-e" title="View models and scores"><strong>MultiPL-E</strong></a> <span class="score-symbol" title="43 scores" aria-label="43 scores">O</span> <a href="https://github.com/nuprl/MultiPL-E" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation (multilingual)</td> <td class="desc">Multilingual code generation and execution benchmark across many programming languages.</td> <td data-sort="2"> <span class="stars" title="Usage: 3/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="269">269</td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4</span> </a> </td> <td data-sort="89.6"> 89.6% </td> </tr> <tr data-bench="multipl-e-humaneval" data-open-source="1"> <td style="text-align:center" data-sort="444">444</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multipl-e-humaneval" class="llm-fu-bench-link" data-bench-id="multipl-e-humaneval" title="View models and scores"><strong>MultiPL-E HumanEval</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://github.com/nuprl/MultiPL-E" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation (multilingual)</td> <td class="desc">MultiPL-E variant of HumanEval tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B</span> </a> </td> <td data-sort="75.2"> 75.2% </td> </tr> <tr data-bench="multipl-e-mbpp" data-open-source="1"> <td style="text-align:center" data-sort="445">445</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=multipl-e-mbpp" class="llm-fu-bench-link" data-bench-id="multipl-e-mbpp" title="View models and scores"><strong>MultiPL-E MBPP</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://github.com/nuprl/MultiPL-E" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code generation (multilingual)</td> <td class="desc">MultiPL-E variant of MBPP tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B</span> </a> </td> <td data-sort="65.7"> 65.7% </td> </tr> <tr data-bench="musr" data-open-source="1"> <td style="text-align:center" data-sort="446">446</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=musr" class="llm-fu-bench-link" data-bench-id="musr" title="View models and scores"><strong>MuSR</strong></a> <span class="score-symbol" title="54 scores" aria-label="54 scores">O</span> <a href="https://arxiv.org/abs/2310.16049" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Reasoning</td> <td class="desc">Multistep Soft Reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ling" data-model-version="Ling 2.0" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ling-flash-2.0"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ling Flash 2.0</span> </a> </td> <td data-sort="82.7"> 82.7% </td> </tr> <tr data-bench="mvbench" data-open-source="0"> <td style="text-align:center" data-sort="447">447</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=mvbench" class="llm-fu-bench-link" data-bench-id="mvbench" title="View models and scores"><strong>MVBench</strong></a> <span class="score-symbol" title="26 scores" aria-label="26 scores">O</span> </td> <td>Video QA</td> <td class="desc">Multi-view or multi-video QA benchmark (MVBench).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="78.1"> 78.1% </td> </tr> <tr data-bench="natural2code" data-open-source="0"> <td style="text-align:center" data-sort="448">448</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=natural2code" class="llm-fu-bench-link" data-bench-id="natural2code" title="View models and scores"><strong>Natural2Code</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Code generation</td> <td class="desc">Natural language to code benchmark for instruction-following synthesis.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.0" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.0-flash"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.0 Flash</span> </a> </td> <td data-sort="92.9"> 92.9% </td> </tr> <tr data-bench="naturalquestions" data-open-source="1"> <td style="text-align:center" data-sort="449">449</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=naturalquestions" class="llm-fu-bench-link" data-bench-id="naturalquestions" title="View models and scores"><strong>NaturalQuestions</strong></a> <span class="score-symbol" title="23 scores" aria-label="23 scores">O</span> <a href="https://ai.google.com/research/NaturalQuestions" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Open-domain QA</td> <td class="desc">Google NQ; real user questions with long/short answers.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Mixtral" data-model-version="Mixtral" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mixtral-8x22b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mixtral 8x22B</span> </a> </td> <td data-sort="40.1"> 40.1% </td> </tr> <tr data-bench="nexus" data-open-source="1"> <td style="text-align:center" data-sort="450">450</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=nexus" class="llm-fu-bench-link" data-bench-id="nexus" title="View models and scores"><strong>Nexus (0-shot)</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Tool use</td> <td class="desc">Nexus tool-use benchmark, zero-shot setting.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B</span> </a> </td> <td data-sort="58.7"> 58.7% </td> </tr> <tr data-bench="nih" data-open-source="1"> <td style="text-align:center" data-sort="451">451</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=nih" class="llm-fu-bench-link" data-bench-id="nih" title="View models and scores"><strong>Needle In A Haystack</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/gkamradt/LLMTest_NeedleInAHaystack" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/gkamradt/LLMTest_NeedleInAHaystack" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Long-context retrieval</td> <td class="desc">Needle In A Haystack test for locating hidden facts in long contexts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MobileLLM" data-model-version="MobileLLM P1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mobilellm-p1-base"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MobileLLM P1 Base</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="nolima-128k" data-open-source="1"> <td style="text-align:center" data-sort="452">452</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=nolima-128k" class="llm-fu-bench-link" data-bench-id="nolima-128k" title="View models and scores"><strong>NoLiMa 128K</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context eval</td> <td class="desc">NoLiMa (No Literal Match) long-context benchmark at 128k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiniCPM" data-model-version="MiniCPM SALA" data-provider="openbmb"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minicpm-sala"> <span class="brand"> <img class="logo no-invert" alt="openbmb logo" data-company="openbmb" data-country="China" src="https://furukama.com/logos/openbmb.png?v=b38b6e7c" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">MiniCPM-SALA</span> </a> </td> <td data-sort="23.9"> 23.9% </td> </tr> <tr data-bench="nolima-32k" data-open-source="1"> <td style="text-align:center" data-sort="453">453</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=nolima-32k" class="llm-fu-bench-link" data-bench-id="nolima-32k" title="View models and scores"><strong>NoLiMa 32K</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context eval</td> <td class="desc">NoLiMa (No Literal Match) long-context benchmark at 32k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiniCPM" data-model-version="MiniCPM SALA" data-provider="openbmb"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minicpm-sala"> <span class="brand"> <img class="logo no-invert" alt="openbmb logo" data-company="openbmb" data-country="China" src="https://furukama.com/logos/openbmb.png?v=b38b6e7c" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">MiniCPM-SALA</span> </a> </td> <td data-sort="54.5"> 54.5% </td> </tr> <tr data-bench="nolima-64k" data-open-source="1"> <td style="text-align:center" data-sort="454">454</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=nolima-64k" class="llm-fu-bench-link" data-bench-id="nolima-64k" title="View models and scores"><strong>NoLiMa 64K</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Long-context eval</td> <td class="desc">NoLiMa (No Literal Match) long-context benchmark at 64k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiniCPM" data-model-version="MiniCPM SALA" data-provider="openbmb"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minicpm-sala"> <span class="brand"> <img class="logo no-invert" alt="openbmb logo" data-company="openbmb" data-country="China" src="https://furukama.com/logos/openbmb.png?v=b38b6e7c" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">MiniCPM-SALA</span> </a> </td> <td data-sort="43.0"> 43.0% </td> </tr> <tr data-bench="nova-63" data-open-source="1"> <td style="text-align:center" data-sort="455">455</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=nova-63" class="llm-fu-bench-link" data-bench-id="nova-63" title="View models and scores"><strong>NOVA-63</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>Multilingual evaluation</td> <td class="desc">Multilingual evaluation benchmark covering 63 languages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="59.1"> 59.1% </td> </tr> <tr data-bench="nuscenes" data-open-source="1"> <td style="text-align:center" data-sort="456">456</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=nuscenes" class="llm-fu-bench-link" data-bench-id="nuscenes" title="View models and scores"><strong>NuScenes</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>3D scene understanding</td> <td class="desc">3D scene understanding and perception benchmark for autonomous driving.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="16.0"> 16.0% </td> </tr> <tr data-bench="objectron" data-open-source="1"> <td style="text-align:center" data-sort="457">457</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=objectron" class="llm-fu-bench-link" data-bench-id="objectron" title="View models and scores"><strong>Objectron</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Object detection</td> <td class="desc">Objectron benchmark for 3D object detection in video captures.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-235b-a22b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-235B-A22B Thinking</span> </a> </td> <td data-sort="71.2"> 71.2% </td> </tr> <tr data-bench="obqa" data-open-source="1"> <td style="text-align:center" data-sort="458">458</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=obqa" class="llm-fu-bench-link" data-bench-id="obqa" title="View models and scores"><strong>OBQA</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Open book QA</td> <td class="desc">OpenBookQA science question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5 Omni" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-omni-3b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5-Omni-3B</span> </a> </td> <td data-sort="76.3"> 76.3% </td> </tr> <tr data-bench="ocnli" data-open-source="1"> <td style="text-align:center" data-sort="459">459</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ocnli" class="llm-fu-bench-link" data-bench-id="ocnli" title="View models and scores"><strong>OCNLI</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Natural language inference (Chinese)</td> <td class="desc">Original Chinese Natural Language Inference benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LLaDA" data-model-version="LLaDA 2.1" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llada2.1-flash-q"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">LLaDA2.1-Flash (Q Mode)</span> </a> </td> <td data-sort="72.8"> 72.8% </td> </tr> <tr data-bench="ocrbench" data-open-source="1"> <td style="text-align:center" data-sort="460">460</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ocrbench" class="llm-fu-bench-link" data-bench-id="ocrbench" title="View models and scores"><strong>OCRBench V2</strong></a> <span class="score-symbol" title="95 scores" aria-label="95 scores">O</span> </td> <td>OCR (vision text extraction)</td> <td class="desc">OCRBench v2 evaluating text extraction from images and documents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-2b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL 2B Instruct</span> </a> </td> <td data-sort="858.0"> 858.0% </td> </tr> <tr data-bench="ocrbench-elo" data-open-source="0"> <td style="text-align:center" data-sort="461">461</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ocrbench-elo" class="llm-fu-bench-link" data-bench-id="ocrbench-elo" title="View models and scores"><strong>OCRBench-ELO</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>OCR (ELO ranking)</td> <td class="desc">OCR benchmark using ELO rating system to rank model performance on text extraction tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="866"> 866 </td> </tr> <tr data-bench="ocrbenchv2-cn" data-open-source="1"> <td style="text-align:center" data-sort="462">462</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ocrbenchv2-cn" class="llm-fu-bench-link" data-bench-id="ocrbenchv2-cn" title="View models and scores"><strong>OCRBenchV2 (CN)</strong></a> <span class="score-symbol" title="26 scores" aria-label="26 scores">O</span> </td> <td>OCR (Chinese)</td> <td class="desc">OCRBenchV2 Chinese subset assessing OCR performance on Chinese-language documents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ovis" data-model-version="Ovis 2.6" data-provider="alibaba"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ovis2.6-30b-a3b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ovis2.6-30B-A3B</span> </a> </td> <td data-sort="67.1"> 67.1% </td> </tr> <tr data-bench="ocrbenchv2-en" data-open-source="1"> <td style="text-align:center" data-sort="463">463</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ocrbenchv2-en" class="llm-fu-bench-link" data-bench-id="ocrbenchv2-en" title="View models and scores"><strong>OCRBenchV2 (EN)</strong></a> <span class="score-symbol" title="25 scores" aria-label="25 scores">O</span> </td> <td>OCR (English)</td> <td class="desc">OCRBenchV2 English subset evaluating OCR accuracy on English documents and layouts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ovis" data-model-version="Ovis 2.6" data-provider="alibaba"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ovis2.6-30b-a3b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ovis2.6-30B-A3B</span> </a> </td> <td data-sort="72.6"> 72.6% </td> </tr> <tr data-bench="ocrreasoning" data-open-source="0"> <td style="text-align:center" data-sort="464">464</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ocrreasoning" class="llm-fu-bench-link" data-bench-id="ocrreasoning" title="View models and scores"><strong>OCRReasoning</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>OCR reasoning</td> <td class="desc">OCR reasoning benchmark combining text extraction with multi-step reasoning over documents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="70.8"> 70.8% </td> </tr> <tr data-bench="octocodingbench" data-open-source="0"> <td style="text-align:center" data-sort="465">465</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=octocodingbench" class="llm-fu-bench-link" data-bench-id="octocodingbench" title="View models and scores"><strong>OctoCodingBench</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Code generation</td> <td class="desc">Coding benchmark across multi-language programming tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="36.2"> 36.2% </td> </tr> <tr data-bench="odinw-13" data-open-source="1"> <td style="text-align:center" data-sort="466">466</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=odinw-13" class="llm-fu-bench-link" data-bench-id="odinw-13" title="View models and scores"><strong>ODinW-13</strong></a> <span class="score-symbol" title="18 scores" aria-label="18 scores">O</span> </td> <td>Object detection (in the wild)</td> <td class="desc">Object Detection in the Wild benchmark covering 13 real-world domains.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-4b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-4B-Instruct</span> </a> </td> <td data-sort="48.2"> 48.2% </td> </tr> <tr data-bench="odyssey-math" data-open-source="1"> <td style="text-align:center" data-sort="467">467</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=odyssey-math" class="llm-fu-bench-link" data-bench-id="odyssey-math" title="View models and scores"><strong>Odyssey Math</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Math reasoning</td> <td class="desc">Odyssey multi-step math benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Mathstral" data-model-version="Mathstral" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mathstral-7b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Mathstral 7B</span> </a> </td> <td data-sort="37.2"> 37.2% </td> </tr> <tr data-bench="oibench-en" data-open-source="0"> <td style="text-align:center" data-sort="468">468</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=oibench-en" class="llm-fu-bench-link" data-bench-id="oibench-en" title="View models and scores"><strong>OIBench EN</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Code generation</td> <td class="desc">English subset of OIBench for code generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="58.2"> 58.2% </td> </tr> <tr data-bench="ojbench" data-open-source="0"> <td style="text-align:center" data-sort="469">469</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ojbench" class="llm-fu-bench-link" data-bench-id="ojbench" title="View models and scores"><strong>OJBench</strong></a> <span class="score-symbol" title="42 scores" aria-label="42 scores">O</span> </td> <td>Code generation (online judge)</td> <td class="desc">Programming problems evaluated via online judge-style execution.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="68.5"> 68.5% </td> </tr> <tr data-bench="olmocr-bench" data-open-source="1"> <td style="text-align:center" data-sort="470">470</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=olmocr-bench" class="llm-fu-bench-link" data-bench-id="olmocr-bench" title="View models and scores"><strong>olmOCR-Bench</strong></a> <span class="score-symbol" title="24 scores" aria-label="24 scores">O</span> </td> <td>Document OCR</td> <td class="desc">olmOCR benchmark assessing OCR fidelity and structured extraction on complex document pages.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Chandra OCR" data-model-version="Chandra OCR" data-provider="chandra"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=chandra-ocr-0.1.0"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Chandra OCR 0.1.0</span> </a> </td> <td data-sort="83.1"> 83.1% </td> </tr> <tr data-bench="olympiadbench" data-open-source="1"> <td style="text-align:center" data-sort="471">471</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=olympiadbench" class="llm-fu-bench-link" data-bench-id="olympiadbench" title="View models and scores"><strong>OlympiadBench</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Math (olympiad)</td> <td class="desc">Advanced mathematics olympiad-style problem benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-instruct-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Instruct-2507</span> </a> </td> <td data-sort="77.6"> 77.6% </td> </tr> <tr data-bench="olympic-arena" data-open-source="1"> <td style="text-align:center" data-sort="472">472</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=olympic-arena" class="llm-fu-bench-link" data-bench-id="olympic-arena" title="View models and scores"><strong>OlympicArena</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Math (competition)</td> <td class="desc">Olympiad-style mathematics reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3</span> </a> </td> <td data-sort="76.2"> 76.2% </td> </tr> <tr data-bench="omega" data-open-source="1"> <td style="text-align:center" data-sort="473">473</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omega" class="llm-fu-bench-link" data-bench-id="omega" title="View models and scores"><strong>OMEGA</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Math (advanced)</td> <td class="desc">OMEGA olympiad-grade mathematics reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="OLMo" data-model-version="OLMo 3" data-provider="allenai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=olmo-3-think-32b"> <span class="brand"> <img class="logo" alt="allenai logo" data-company="allenai" data-country="United States" src="https://furukama.com/logos/allenai.svg?v=27b5cbbf" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">OLMo-3-Think-32B</span> </a> </td> <td data-sort="50.8"> 50.8% </td> </tr> <tr data-bench="omni-math" data-open-source="1"> <td style="text-align:center" data-sort="474">474</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omni-math" class="llm-fu-bench-link" data-bench-id="omni-math" title="View models and scores"><strong>Omni-MATH</strong></a> <span class="score-symbol" title="22 scores" aria-label="22 scores">O</span> </td> <td>Math reasoning</td> <td class="desc">Omni-MATH benchmark covering diverse math reasoning tasks across difficulty levels.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ling" data-model-version="Ling 1T" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ling-1t"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ling 1T</span> </a> </td> <td data-sort="74.5"> 74.5% </td> </tr> <tr data-bench="omni-math-hard" data-open-source="0"> <td style="text-align:center" data-sort="475">475</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omni-math-hard" class="llm-fu-bench-link" data-bench-id="omni-math-hard" title="View models and scores"><strong>Omni-MATH-HARD</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> </td> <td>Math</td> <td class="desc">Challenging math benchmark (Omni-MATH-HARD).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="73.6"> 73.6% </td> </tr> <tr data-bench="omnidocbench" data-open-source="1"> <td style="text-align:center" data-sort="476">476</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omnidocbench" class="llm-fu-bench-link" data-bench-id="omnidocbench" title="View models and scores"><strong>OmniDocBench</strong></a> <span class="score-symbol" title="61 scores" aria-label="61 scores">O</span> </td> <td>Document understanding</td> <td class="desc">Document understanding benchmark covering multi-page layouts, tables, and charts for robust question answering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek OCR" data-model-version="DeepSeek OCR" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gundam-m"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Gundam-M</span> </a> </td> <td data-sort="12.3"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 12.3%</span> </td> </tr> <tr data-bench="omnidocbench-1.5" data-open-source="1"> <td style="text-align:center" data-sort="477">477</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omnidocbench-1.5" class="llm-fu-bench-link" data-bench-id="omnidocbench-1.5" title="View models and scores"><strong>OmniDocBench 1.5</strong></a> <span class="score-symbol" title="26 scores" aria-label="26 scores">O</span> </td> <td>OCR</td> <td class="desc">Document understanding benchmark v1.5 with OCR evaluation. Overall Edit Distance metric, lower is better.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Dolphin" data-model-version="Dolphin V2" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=dolphin-v2"> <span class="brand"> <img class="logo" alt="microsoft logo" data-company="microsoft" data-country="United States" src="https://furukama.com/logos/microsoft.svg?v=acf5d3c0" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Dolphin V2</span> </a> </td> <td data-sort="0.1"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 0.1%</span> </td> </tr> <tr data-bench="omnidocbench-cn" data-open-source="1"> <td style="text-align:center" data-sort="478">478</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omnidocbench-cn" class="llm-fu-bench-link" data-bench-id="omnidocbench-cn" title="View models and scores"><strong>OmniDocBench-CN</strong></a> <span class="score-symbol" title="26 scores" aria-label="26 scores">O</span> </td> <td>Document understanding (Chinese)</td> <td class="desc">Chinese subset of OmniDocBench focusing on OCR-grounded document comprehension and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="PPStructure" data-model-version="PPStructure" data-provider="paddlepaddle"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ppstructure-v3"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">PPStructure v3</span> </a> </td> <td data-sort="13.6"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 13.6%</span> </td> </tr> <tr data-bench="omnimmi" data-open-source="0"> <td style="text-align:center" data-sort="479">479</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omnimmi" class="llm-fu-bench-link" data-bench-id="omnimmi" title="View models and scores"><strong>OmniMMI</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Multimodal interaction</td> <td class="desc">OmniMMI benchmark for multimodal interaction across video streams.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="53.0"> 53.0% </td> </tr> <tr data-bench="omnispatial" data-open-source="1"> <td style="text-align:center" data-sort="480">480</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=omnispatial" class="llm-fu-bench-link" data-bench-id="omnispatial" title="View models and scores"><strong>OmniSpatial</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Spatial reasoning</td> <td class="desc">Spatial understanding and reasoning benchmark (OmniSpatial).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.6V</span> </a> </td> <td data-sort="52.0"> 52.0% </td> </tr> <tr data-bench="oneig-bench-en" data-open-source="0"> <td style="text-align:center" data-sort="481">481</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=oneig-bench-en" class="llm-fu-bench-link" data-bench-id="oneig-bench-en" title="View models and scores"><strong>OneIG-Bench EN</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Text-to-image</td> <td class="desc">OneIG-Bench English subset score for text-to-image generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nano Banana" data-model-version="Nano Banana" data-provider="unknown"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nano-banana-2.0"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Nano Banana 2.0</span> </a> </td> <td data-sort="0.6"> 0.6% </td> </tr> <tr data-bench="oneig-bench-zh" data-open-source="0"> <td style="text-align:center" data-sort="482">482</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=oneig-bench-zh" class="llm-fu-bench-link" data-bench-id="oneig-bench-zh" title="View models and scores"><strong>OneIG-Bench ZH</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Text-to-image</td> <td class="desc">OneIG-Bench Chinese subset score for text-to-image generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nano Banana" data-model-version="Nano Banana" data-provider="unknown"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nano-banana-2.0"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Nano Banana 2.0</span> </a> </td> <td data-sort="0.6"> 0.6% </td> </tr> <tr data-bench="online-mind2web" data-open-source="0"> <td style="text-align:center" data-sort="483">483</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=online-mind2web" class="llm-fu-bench-link" data-bench-id="online-mind2web" title="View models and scores"><strong>Online-Mind2web</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Web automation</td> <td class="desc">Online web automation and task execution benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="85.9"> 85.9% </td> </tr> <tr data-bench="open-rewrite" data-open-source="1"> <td style="text-align:center" data-sort="484">484</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=open-rewrite" class="llm-fu-bench-link" data-bench-id="open-rewrite" title="View models and scores"><strong>Open Rewrite</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Instruction following</td> <td class="desc">Rewrite benchmark assessing open-ended editing and directive-following quality.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MobileLLM" data-model-version="MobileLLM" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mobilellm-p1"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MobileLLM P1</span> </a> </td> <td data-sort="51.0"> 51.0% </td> </tr> <tr data-bench="openbookqa" data-open-source="1"> <td style="text-align:center" data-sort="485">485</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=openbookqa" class="llm-fu-bench-link" data-bench-id="openbookqa" title="View models and scores"><strong>OpenBookQA</strong></a> <span class="score-symbol" title="30 scores" aria-label="30 scores">O</span> <a href="https://github.com/allenai/OpenBookQA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Science QA</td> <td class="desc">Open-book multiple choice science questions with supporting facts.</td> <td data-sort="2"> <span class="stars" title="Usage: 3/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="128">128</td> <td class="leader-cell" data-model-family="Hermes" data-model-version="Hermes 4.3" data-provider="nousresearch"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=hermes-4.3-36b-pyche"> <span class="brand"> <img class="logo no-invert" alt="nousresearch logo" data-company="nousresearch" data-country="United States" src="https://furukama.com/logos/nousresearch.svg?v=8d818340" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Hermes 4.3 36B Pyche</span> </a> </td> <td data-sort="96.6"> 96.6% </td> </tr> <tr data-bench="openrewrite-eval" data-open-source="1"> <td style="text-align:center" data-sort="486">486</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=openrewrite-eval" class="llm-fu-bench-link" data-bench-id="openrewrite-eval" title="View models and scores"><strong>OpenRewrite-Eval</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Rewrite quality</td> <td class="desc">OpenRewrite evaluation; micro-averaged RougeL.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen2.5-1.5b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5 1.5B Instruct</span> </a> </td> <td data-sort="46.9"> 46.9% </td> </tr> <tr data-bench="optmath" data-open-source="1"> <td style="text-align:center" data-sort="487">487</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=optmath" class="llm-fu-bench-link" data-bench-id="optmath" title="View models and scores"><strong>OptMATH</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Math optimization reasoning</td> <td class="desc">OptMATH benchmark targeting challenging math optimization and problem-solving tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ling" data-model-version="Ling 1T" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ling-1t"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ling 1T</span> </a> </td> <td data-sort="57.7"> 57.7% </td> </tr> <tr data-bench="order-15-items" data-open-source="1"> <td style="text-align:center" data-sort="488">488</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=order-15-items" class="llm-fu-bench-link" data-bench-id="order-15-items" title="View models and scores"><strong>Order 15 Items</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>List ordering</td> <td class="desc">Ordering benchmark requiring models to sequence 15 items correctly.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="K2" data-model-version="K2 V2" data-provider="mbzuai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k2-v2"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K2-V2</span> </a> </td> <td data-sort="87.6"> 87.6% </td> </tr> <tr data-bench="order-30-items" data-open-source="1"> <td style="text-align:center" data-sort="489">489</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=order-30-items" class="llm-fu-bench-link" data-bench-id="order-30-items" title="View models and scores"><strong>Order 30 Items</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>List ordering (long)</td> <td class="desc">Ordering benchmark requiring models to sequence 30 items correctly.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="K2" data-model-version="K2 V2" data-provider="mbzuai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=k2-v2"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">K2-V2</span> </a> </td> <td data-sort="40.3"> 40.3% </td> </tr> <tr data-bench="osworld" data-open-source="0"> <td style="text-align:center" data-sort="490">490</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=osworld" class="llm-fu-bench-link" data-bench-id="osworld" title="View models and scores"><strong>OSWorld</strong></a> <span class="score-symbol" title="26 scores" aria-label="26 scores">O</span> </td> <td>GUI agents</td> <td class="desc">Agentic GUI task completion and grounding on desktop environments.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="72.7"> 72.7% </td> </tr> <tr data-bench="osworld-g" data-open-source="1"> <td style="text-align:center" data-sort="491">491</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=osworld-g" class="llm-fu-bench-link" data-bench-id="osworld-g" title="View models and scores"><strong>OSWorld-G</strong></a> <span class="score-symbol" title="29 scores" aria-label="29 scores">O</span> </td> <td>GUI agents</td> <td class="desc">OSWorld-G center accuracy (no_refusal).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Holo" data-model-version="Holo1.5" data-provider="hcompany"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=holo1.5-72b"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Holo1.5-72B</span> </a> </td> <td data-sort="71.8"> 71.8% </td> </tr> <tr data-bench="osworld-verified" data-open-source="0"> <td style="text-align:center" data-sort="492">492</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=osworld-verified" class="llm-fu-bench-link" data-bench-id="osworld-verified" title="View models and scores"><strong>OSWorld Verified</strong></a> <span class="score-symbol" title="17 scores" aria-label="17 scores">O</span> </td> <td>GUI agents</td> <td class="desc">Verified subset of OSWorld GUI agent benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="72.7"> 72.7% </td> </tr> <tr data-bench="osworld2" data-open-source="1"> <td style="text-align:center" data-sort="493">493</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=osworld2" class="llm-fu-bench-link" data-bench-id="osworld2" title="View models and scores"><strong>OSWorld2</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>GUI agents</td> <td class="desc">Second-generation OSWorld GUI agent benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.5V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.5v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.5V</span> </a> </td> <td data-sort="35.8"> 35.8% </td> </tr> <tr data-bench="ovbench" data-open-source="0"> <td style="text-align:center" data-sort="494">494</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ovbench" class="llm-fu-bench-link" data-bench-id="ovbench" title="View models and scores"><strong>OVBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Open-vocabulary streaming</td> <td class="desc">Open-vocabulary benchmark for streaming video understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="65.1"> 65.1% </td> </tr> <tr data-bench="ovobench" data-open-source="0"> <td style="text-align:center" data-sort="495">495</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ovobench" class="llm-fu-bench-link" data-bench-id="ovobench" title="View models and scores"><strong>OVOBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Streaming video QA</td> <td class="desc">Streaming video QA benchmark with open-vocabulary queries.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="72.6"> 72.6% </td> </tr> <tr data-bench="paper-bench-code-dev" data-open-source="0"> <td style="text-align:center" data-sort="496">496</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=paper-bench-code-dev" class="llm-fu-bench-link" data-bench-id="paper-bench-code-dev" title="View models and scores"><strong>PaperBench Code-Dev</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Code understanding</td> <td class="desc">PaperBench developer subset measuring code reasoning accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4</span> </a> </td> <td data-sort="43.3"> 43.3% </td> </tr> <tr data-bench="paperbench" data-open-source="0"> <td style="text-align:center" data-sort="497">497</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=paperbench" class="llm-fu-bench-link" data-bench-id="paperbench" title="View models and scores"><strong>PaperBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Research paper understanding</td> <td class="desc">Benchmark for understanding and reasoning over research papers.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5 Thinking</span> </a> </td> <td data-sort="72.9"> 72.9% </td> </tr> <tr data-bench="phybench" data-open-source="0"> <td style="text-align:center" data-sort="498">498</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=phybench" class="llm-fu-bench-link" data-bench-id="phybench" title="View models and scores"><strong>PHYBench</strong></a> <span class="score-symbol" title="24 scores" aria-label="24 scores">O</span> </td> <td>Physics reasoning</td> <td class="desc">Physics reasoning and calculation benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="80.0"> 80.0% </td> </tr> <tr data-bench="phyx" data-open-source="1"> <td style="text-align:center" data-sort="499">499</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=phyx" class="llm-fu-bench-link" data-bench-id="phyx" title="View models and scores"><strong>PhyX</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Physics reasoning (multimodal)</td> <td class="desc">Multimodal physics reasoning benchmark (PhyX).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step3-VL" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step3-vl-10b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step3-VL-10B</span> </a> </td> <td data-sort="59.5"> 59.5% </td> </tr> <tr data-bench="piqa" data-open-source="1"> <td style="text-align:center" data-sort="500">500</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=piqa" class="llm-fu-bench-link" data-bench-id="piqa" title="View models and scores"><strong>PIQA</strong></a> <span class="score-symbol" title="73 scores" aria-label="73 scores">O</span> <a href="https://github.com/ybisk/piqa" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Physical commonsense</td> <td class="desc">Physical commonsense about everyday tasks and object affordances.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LLaDA" data-model-version="LLaDA 2.0" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llada2.0-flash"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">LLaDA2.0 Flash</span> </a> </td> <td data-sort="96.5"> 96.5% </td> </tr> <tr data-bench="pixmocount" data-open-source="1"> <td style="text-align:center" data-sort="501">501</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=pixmocount" class="llm-fu-bench-link" data-bench-id="pixmocount" title="View models and scores"><strong>PixmoCount</strong></a> <span class="score-symbol" title="53 scores" aria-label="53 scores">O</span> <a href="https://huggingface.co/datasets/allenai/pixmo-count" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2409.17146v2" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Visual counting</td> <td class="desc">Counting objects/instances in images (PixmoCount).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Eagle" data-model-version="Eagle 2.5" data-provider="unknown"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=eagle2.5-8b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Eagle2.5-8B</span> </a> </td> <td data-sort="90.2"> 90.2% </td> </tr> <tr data-bench="pmc-vqa" data-open-source="1"> <td style="text-align:center" data-sort="502">502</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=pmc-vqa" class="llm-fu-bench-link" data-bench-id="pmc-vqa" title="View models and scores"><strong>PMC-VQA</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Medical VQA</td> <td class="desc">PubMed Central visual question answering benchmark for biomedical images.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="64.2"> 64.2% </td> </tr> <tr data-bench="point-bench" data-open-source="0"> <td style="text-align:center" data-sort="503">503</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=point-bench" class="llm-fu-bench-link" data-bench-id="point-bench" title="View models and scores"><strong>Point-Bench</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Pointing and counting</td> <td class="desc">Benchmark for pointing and counting objects in images.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="85.5"> 85.5% </td> </tr> <tr data-bench="polymath" data-open-source="0"> <td style="text-align:center" data-sort="504">504</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=polymath" class="llm-fu-bench-link" data-bench-id="polymath" title="View models and scores"><strong>PolyMATH</strong></a> <span class="score-symbol" title="42 scores" aria-label="42 scores">O</span> </td> <td>Math reasoning</td> <td class="desc">Polyglot mathematics benchmark assessing cross-topic math reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="81.6"> 81.6% </td> </tr> <tr data-bench="pope" data-open-source="1"> <td style="text-align:center" data-sort="505">505</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=pope" class="llm-fu-bench-link" data-bench-id="pope" title="View models and scores"><strong>POPE</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> <a href="https://arxiv.org/abs/2305.10355" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/AoiDragon/POPE" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Hallucination detection</td> <td class="desc">Vision-language hallucination benchmark focusing on object existence verification.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="InternVL 3" data-provider="opengvlab"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=internvl3-2b"> <span class="brand"> <img class="logo no-invert" alt="opengvlab logo" data-company="opengvlab" data-country="China" src="https://furukama.com/logos/opengvlab.png?v=e14fd644" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">InternVL3-2B</span> </a> </td> <td data-sort="90.1"> 90.1% </td> </tr> <tr data-bench="popqa" data-open-source="1"> <td style="text-align:center" data-sort="506">506</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=popqa" class="llm-fu-bench-link" data-bench-id="popqa" title="View models and scores"><strong>PopQA</strong></a> <span class="score-symbol" title="41 scores" aria-label="41 scores">O</span> <a href="https://github.com/facebookresearch/PopQA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2211.07239" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/facebookresearch/PopQA" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Knowledge / QA</td> <td class="desc">Open-domain popular culture question answering benchmark testing long-tail factual recall.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Tulu 3" data-provider="allenai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-tulu-3-405b-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 Tulu 3 405B SFT</span> </a> </td> <td data-sort="55.7"> 55.7% </td> </tr> <tr data-bench="posttrainbench" data-open-source="0"> <td style="text-align:center" data-sort="507">507</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=posttrainbench" class="llm-fu-bench-link" data-bench-id="posttrainbench" title="View models and scores"><strong>PostTrainBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://posttrainbench.com/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://posttrainbench.com/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/aisa-group/PostTrainBench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Post-training automation</td> <td class="desc">Measures how well AI agents can post-train base LLMs under fixed compute/time constraints; average score across AIME 2025, BFCL, GPQA Main, GSM8K, and HumanEval.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.1 Codex" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.1-codex-max"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.1 Codex-Max</span> </a> </td> <td data-sort="34.9"> 34.9% </td> </tr> <tr data-bench="prdbench" data-open-source="1"> <td style="text-align:center" data-sort="508">508</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=prdbench" class="llm-fu-bench-link" data-bench-id="prdbench" title="View models and scores"><strong>PRDBench</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Agentic coding</td> <td class="desc">Product Requirements Document benchmark for evaluating agentic coding capabilities.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LongCat" data-model-version="LongCat-Flash-Lite" data-provider="meituan"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=longcat-flash-lite"> <span class="brand"> <img class="logo no-invert" alt="meituan logo" data-company="meituan" data-country="China" src="https://furukama.com/logos/meituan.svg?v=45908c84" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">LongCat-Flash-Lite</span> </a> </td> <td data-sort="39.6"> 39.6% </td> </tr> <tr data-bench="procbench" data-open-source="0"> <td style="text-align:center" data-sort="509">509</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=procbench" class="llm-fu-bench-link" data-bench-id="procbench" title="View models and scores"><strong>ProcBench</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Procedural reasoning</td> <td class="desc">Procedural reasoning benchmark evaluating step-by-step logical reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 2.0" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed2.0-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed2.0 Pro</span> </a> </td> <td data-sort="96.6"> 96.6% </td> </tr> <tr data-bench="prontoqa" data-open-source="1"> <td style="text-align:center" data-sort="510">510</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=prontoqa" class="llm-fu-bench-link" data-bench-id="prontoqa" title="View models and scores"><strong>PrOntoQA</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> </td> <td>Logical reasoning</td> <td class="desc">Probing ontological reasoning via question answering.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ling" data-model-version="Ling 2.0" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ling-flash-2.0"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ling Flash 2.0</span> </a> </td> <td data-sort="97.9"> 97.9% </td> </tr> <tr data-bench="proofbench-advanced" data-open-source="0"> <td style="text-align:center" data-sort="511">511</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=proofbench-advanced" class="llm-fu-bench-link" data-bench-id="proofbench-advanced" title="View models and scores"><strong>ProofBench Advanced</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Mathematical proofs (advanced)</td> <td class="desc">Advanced mathematical proof benchmark covering complex theorem proving tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini Deep Think" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-deep-think-imo-gold"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini Deep Think (IMO Gold)</span> </a> </td> <td data-sort="65.7"> 65.7% </td> </tr> <tr data-bench="proofbench-basic" data-open-source="1"> <td style="text-align:center" data-sort="512">512</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=proofbench-basic" class="llm-fu-bench-link" data-bench-id="proofbench-basic" title="View models and scores"><strong>ProofBench Basic</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Mathematical proofs</td> <td class="desc">Entry-level mathematical proof benchmarking set.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeekMath" data-model-version="DeepSeekMath V2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseekmath-v2-heavy"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeekMath-V2-Heavy</span> </a> </td> <td data-sort="99.0"> 99.0% </td> </tr> <tr data-bench="protocolqa" data-open-source="0"> <td style="text-align:center" data-sort="513">513</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=protocolqa" class="llm-fu-bench-link" data-bench-id="protocolqa" title="View models and scores"><strong>ProtocolQA</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Protocol understanding and QA</td> <td class="desc">Protocol question answering benchmark evaluating understanding of scientific protocols and procedures.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4.1" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4.1-thinking"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4.1 (Thinking)</span> </a> </td> <td data-sort="79.0"> 79.0% </td> </tr> <tr data-bench="quac" data-open-source="1"> <td style="text-align:center" data-sort="514">514</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=quac" class="llm-fu-bench-link" data-bench-id="quac" title="View models and scores"><strong>QuAC</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://quac.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Conversational QA</td> <td class="desc">Question answering in context.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b-base"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B Base</span> </a> </td> <td data-sort="53.6"> 53.6% </td> </tr> <tr data-bench="quality" data-open-source="1"> <td style="text-align:center" data-sort="515">515</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=quality" class="llm-fu-bench-link" data-bench-id="quality" title="View models and scores"><strong>QuALITY</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://github.com/nyu-mll/quality" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2112.08608" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/nyu-mll/quality" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Long-context reading comprehension</td> <td class="desc">Long-document multiple-choice reading comprehension benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="48.8"> 48.8% </td> </tr> <tr data-bench="race" data-open-source="1"> <td style="text-align:center" data-sort="516">516</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=race" class="llm-fu-bench-link" data-bench-id="race" title="View models and scores"><strong>RACE</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="http://www.cs.cmu.edu/~glai1/data/race/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Reading comprehension</td> <td class="desc">English exams for middle and high school.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nemotron-3-nano-30b-a3b-base"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Nemotron-3-Nano-30B-A3B-Base</span> </a> </td> <td data-sort="88.0"> 88.0% </td> </tr> <tr data-bench="random-complex-tasks" data-open-source="1"> <td style="text-align:center" data-sort="517">517</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=random-complex-tasks" class="llm-fu-bench-link" data-bench-id="random-complex-tasks" title="View models and scores"><strong>Random Complex Tasks</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Agentic tasks (random)</td> <td class="desc">Randomly constructed complex task environments for agent generalization.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LongCat" data-model-version="LongCat-Flash" data-provider="meituan"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=longcat-flash-thinking-2601"> <span class="brand"> <img class="logo no-invert" alt="meituan logo" data-company="meituan" data-country="China" src="https://furukama.com/logos/meituan.svg?v=45908c84" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">LongCat-Flash-Thinking-2601</span> </a> </td> <td data-sort="35.8"> 35.8% </td> </tr> <tr data-bench="realbench" data-open-source="0"> <td style="text-align:center" data-sort="518">518</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=realbench" class="llm-fu-bench-link" data-bench-id="realbench" title="View models and scores"><strong>Realbench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Web browsing</td> <td class="desc">Real-world browsing and QA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="49.1"> 49.1% </td> </tr> <tr data-bench="realworldqa" data-open-source="1"> <td style="text-align:center" data-sort="519">519</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=realworldqa" class="llm-fu-bench-link" data-bench-id="realworldqa" title="View models and scores"><strong>RealWorldQA</strong></a> <span class="score-symbol" title="108 scores" aria-label="108 scores">O</span> <a href="https://realworldqa.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2410.14332" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/ZhuoyuWei/RealWorldQAPlatform" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Real-world visual QA</td> <td class="desc">Visual question answering with real-world images and scenarios.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-122b-a10b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-122B-A10B</span> </a> </td> <td data-sort="85.1"> 85.1% </td> </tr> <tr data-bench="ref-l4-test" data-open-source="1"> <td style="text-align:center" data-sort="520">520</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ref-l4-test" class="llm-fu-bench-link" data-bench-id="ref-l4-test" title="View models and scores"><strong>Ref-L4 (test)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://huggingface.co/datasets/JierunChen/Ref-L4" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Referring expressions</td> <td class="desc">Ref-L4 referring expression comprehension on the test split.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.6V</span> </a> </td> <td data-sort="88.9"> 88.9% </td> </tr> <tr data-bench="refcoco" data-open-source="1"> <td style="text-align:center" data-sort="521">521</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=refcoco" class="llm-fu-bench-link" data-bench-id="refcoco" title="View models and scores"><strong>RefCOCO</strong></a> <span class="score-symbol" title="28 scores" aria-label="28 scores">O</span> <a href="https://huggingface.co/datasets/JierunChen/Ref-L4" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Referring expressions</td> <td class="desc">RefCOCO average accuracy at IoU 0.5 (val).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="InternVL 3.5" data-provider="opengvlab"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=internvl3.5-4b"> <span class="brand"> <img class="logo no-invert" alt="opengvlab logo" data-company="opengvlab" data-country="China" src="https://furukama.com/logos/opengvlab.png?v=e14fd644" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">InternVL3.5-4B</span> </a> </td> <td data-sort="92.4"> 92.4% </td> </tr> <tr data-bench="refcocog" data-open-source="1"> <td style="text-align:center" data-sort="522">522</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=refcocog" class="llm-fu-bench-link" data-bench-id="refcocog" title="View models and scores"><strong>RefCOCOg</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://huggingface.co/datasets/JierunChen/Ref-L4" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Referring expressions</td> <td class="desc">RefCOCOg average accuracy at IoU 0.5 (val).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Moondream" data-model-version="Moondream 3" data-provider="moondream"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=moondream-9b-a2b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Moondream-9B-A2B</span> </a> </td> <td data-sort="88.6"> 88.6% </td> </tr> <tr data-bench="refcocoplus" data-open-source="1"> <td style="text-align:center" data-sort="523">523</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=refcocoplus" class="llm-fu-bench-link" data-bench-id="refcocoplus" title="View models and scores"><strong>RefCOCO&#43;</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://huggingface.co/datasets/JierunChen/Ref-L4" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Referring expressions</td> <td class="desc">RefCOCO&#43; accuracy at IoU 0.5 on the val split.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Moondream" data-model-version="Moondream 3" data-provider="moondream"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=moondream-9b-a2b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Moondream-9B-A2B</span> </a> </td> <td data-sort="81.8"> 81.8% </td> </tr> <tr data-bench="refspatialbench" data-open-source="1"> <td style="text-align:center" data-sort="524">524</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=refspatialbench" class="llm-fu-bench-link" data-bench-id="refspatialbench" title="View models and scores"><strong>RefSpatialBench</strong></a> <span class="score-symbol" title="27 scores" aria-label="27 scores">O</span> </td> <td>Spatial reasoning</td> <td class="desc">Reference spatial understanding benchmark covering spatial grounding tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="73.6"> 73.6% </td> </tr> <tr data-bench="refusalbench" data-open-source="1"> <td style="text-align:center" data-sort="525">525</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=refusalbench" class="llm-fu-bench-link" data-bench-id="refusalbench" title="View models and scores"><strong>RefusalBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Safety / refusal</td> <td class="desc">Safety-oriented refusal and policy adherence benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Hermes" data-model-version="Hermes 4.3" data-provider="nousresearch"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=hermes-4.3-36b-pyche"> <span class="brand"> <img class="logo no-invert" alt="nousresearch logo" data-company="nousresearch" data-country="United States" src="https://furukama.com/logos/nousresearch.svg?v=8d818340" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Hermes 4.3 36B Pyche</span> </a> </td> <td data-sort="72.3"> 72.3% </td> </tr> <tr data-bench="remi" data-open-source="1"> <td style="text-align:center" data-sort="526">526</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=remi" class="llm-fu-bench-link" data-bench-id="remi" title="View models and scores"><strong>ReMI</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Multimodal reasoning</td> <td class="desc">Reasoning over multimodal inputs (ReMI).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step3-VL" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step3-vl-10b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step3-VL-10B</span> </a> </td> <td data-sort="67.3"> 67.3% </td> </tr> <tr data-bench="repobench" data-open-source="0"> <td style="text-align:center" data-sort="527">527</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=repobench" class="llm-fu-bench-link" data-bench-id="repobench" title="View models and scores"><strong>RepoBench</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Code understanding</td> <td class="desc">Repository-level code comprehension and reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-4.5-sonnet"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5</span> </a> </td> <td data-sort="83.8"> 83.8% </td> </tr> <tr data-bench="researchrubrics" data-open-source="1"> <td style="text-align:center" data-sort="528">528</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=researchrubrics" class="llm-fu-bench-link" data-bench-id="researchrubrics" title="View models and scores"><strong>ResearchRubrics</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Research evaluation</td> <td class="desc">Benchmark evaluating model ability to conduct research and synthesize findings.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step 3.5 Flash" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step-3.5-flash-20260204"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step-3.5 Flash 20260204</span> </a> </td> <td data-sort="65.3"> 65.3% </td> </tr> <tr data-bench="robospatialhome" data-open-source="1"> <td style="text-align:center" data-sort="529">529</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=robospatialhome" class="llm-fu-bench-link" data-bench-id="robospatialhome" title="View models and scores"><strong>RoboSpatialHome</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Embodied spatial understanding</td> <td class="desc">RoboSpatialHome benchmark for embodied spatial reasoning in domestic environments.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-235b-a22b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-235B-A22B Thinking</span> </a> </td> <td data-sort="73.9"> 73.9% </td> </tr> <tr data-bench="roo-code-evals" data-open-source="0"> <td style="text-align:center" data-sort="530">530</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=roo-code-evals" class="llm-fu-bench-link" data-bench-id="roo-code-evals" title="View models and scores"><strong>Roo Code Evals</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> <a href="https://roocode.com/evals" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://roocode.com/evals" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/RooCodeInc/Roo-Code-Evals" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Code assistant eval</td> <td class="desc">Community-maintained coding evals and leaderboard by Roo Code.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 mini</span> </a> </td> <td data-sort="99.0"> 99.0% </td> </tr> <tr data-bench="ruler-100-1m" data-open-source="1"> <td style="text-align:center" data-sort="531">531</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-100-1m" class="llm-fu-bench-link" data-bench-id="ruler-100-1m" title="View models and scores"><strong>RULER-100 @1M</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER-100 evaluation at a 1M context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nvidia-nemotron-3-nano-30b-a3b-bf16"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">NVIDIA-Nemotron-3-Nano-30B-A3B-BF16</span> </a> </td> <td data-sort="86.3"> 86.3% </td> </tr> <tr data-bench="ruler-100-256k" data-open-source="1"> <td style="text-align:center" data-sort="532">532</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-100-256k" class="llm-fu-bench-link" data-bench-id="ruler-100-256k" title="View models and scores"><strong>RULER-100 @256k</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER-100 evaluation at a 256k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nvidia-nemotron-3-nano-30b-a3b-bf16"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">NVIDIA-Nemotron-3-Nano-30B-A3B-BF16</span> </a> </td> <td data-sort="92.9"> 92.9% </td> </tr> <tr data-bench="ruler-100-512k" data-open-source="1"> <td style="text-align:center" data-sort="533">533</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-100-512k" class="llm-fu-bench-link" data-bench-id="ruler-100-512k" title="View models and scores"><strong>RULER-100 @512k</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER-100 evaluation at a 512k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nvidia-nemotron-3-nano-30b-a3b-bf16"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">NVIDIA-Nemotron-3-Nano-30B-A3B-BF16</span> </a> </td> <td data-sort="91.3"> 91.3% </td> </tr> <tr data-bench="ruler-128k" data-open-source="1"> <td style="text-align:center" data-sort="534">534</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-128k" class="llm-fu-bench-link" data-bench-id="ruler-128k" title="View models and scores"><strong>Ruler 128k</strong></a> <span class="score-symbol" title="18 scores" aria-label="18 scores">O</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 128k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 Next" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-next-80b-a3b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Next-80B-A3B-Instruct</span> </a> </td> <td data-sort="96.0"> 96.0% </td> </tr> <tr data-bench="ruler-16k" data-open-source="1"> <td style="text-align:center" data-sort="535">535</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-16k" class="llm-fu-bench-link" data-bench-id="ruler-16k" title="View models and scores"><strong>Ruler 16k</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 16k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen2.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen-2.5-7.6b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen2.5 7.6B</span> </a> </td> <td data-sort="92.2"> 92.2% </td> </tr> <tr data-bench="ruler-1m" data-open-source="1"> <td style="text-align:center" data-sort="536">536</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-1m" class="llm-fu-bench-link" data-bench-id="ruler-1m" title="View models and scores"><strong>Ruler 1M</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 1M context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi Linear" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-linear-instruct"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-Linear-Instruct</span> </a> </td> <td data-sort="94.8"> 94.8% </td> </tr> <tr data-bench="ruler-32k" data-open-source="0"> <td style="text-align:center" data-sort="537">537</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-32k" class="llm-fu-bench-link" data-bench-id="ruler-32k" title="View models and scores"><strong>Ruler 32k</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 32k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Mistral" data-model-version="Mistral 3" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mistral-medium-3"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mistral Medium 3</span> </a> </td> <td data-sort="96.0"> 96.0% </td> </tr> <tr data-bench="ruler-4k" data-open-source="1"> <td style="text-align:center" data-sort="538">538</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-4k" class="llm-fu-bench-link" data-bench-id="ruler-4k" title="View models and scores"><strong>Ruler 4k</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 4k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ministral" data-model-version="Ministral" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ministral-8b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Ministral 8B</span> </a> </td> <td data-sort="96.0"> 96.0% </td> </tr> <tr data-bench="ruler-512k" data-open-source="1"> <td style="text-align:center" data-sort="539">539</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-512k" class="llm-fu-bench-link" data-bench-id="ruler-512k" title="View models and scores"><strong>Ruler 512k</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 512k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-235b-a22b-instruct-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-235B-A22B-Instruct-2507</span> </a> </td> <td data-sort="90.9"> 90.9% </td> </tr> <tr data-bench="ruler-64k" data-open-source="1"> <td style="text-align:center" data-sort="540">540</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-64k" class="llm-fu-bench-link" data-bench-id="ruler-64k" title="View models and scores"><strong>Ruler 64k</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 64k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiniCPM" data-model-version="MiniCPM SALA" data-provider="openbmb"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minicpm-sala"> <span class="brand"> <img class="logo no-invert" alt="openbmb logo" data-company="openbmb" data-country="China" src="https://furukama.com/logos/openbmb.png?v=b38b6e7c" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">MiniCPM-SALA</span> </a> </td> <td data-sort="92.7"> 92.7% </td> </tr> <tr data-bench="ruler-8k" data-open-source="1"> <td style="text-align:center" data-sort="541">541</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=ruler-8k" class="llm-fu-bench-link" data-bench-id="ruler-8k" title="View models and scores"><strong>Ruler 8k</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://github.com/allenai/ruler" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.06654" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Long-context eval</td> <td class="desc">RULER benchmark at 8k context window.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-8b-base"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 8B Base</span> </a> </td> <td data-sort="93.8"> 93.8% </td> </tr> <tr data-bench="rw-search" data-open-source="0"> <td style="text-align:center" data-sort="542">542</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=rw-search" class="llm-fu-bench-link" data-bench-id="rw-search" title="View models and scores"><strong>RW Search</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Agentic search</td> <td class="desc">Real-world search benchmark evaluating retrieval and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-thinking-xhigh"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Thinking XHigh</span> </a> </td> <td data-sort="82.0"> 82.0% </td> </tr> <tr data-bench="salad-bench" data-open-source="1"> <td style="text-align:center" data-sort="543">543</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=salad-bench" class="llm-fu-bench-link" data-bench-id="salad-bench" title="View models and scores"><strong>SALAD-Bench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://github.com/allenai/salad-bench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2410.08628" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/allenai/salad-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety alignment</td> <td class="desc">Safety Alignment and Dangerous-behavior benchmark evaluating harmful assistance and refusal consistency.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Granite" data-model-version="Granite 4.0" data-provider="ibm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=granite-4.0-h-micro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Granite-4.0-H-Micro</span> </a> </td> <td data-sort="96.8"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 96.8%</span> </td> </tr> <tr data-bench="sarena-icon" data-open-source="1"> <td style="text-align:center" data-sort="544">544</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=sarena-icon" class="llm-fu-bench-link" data-bench-id="sarena-icon" title="View models and scores"><strong>SArena (Icon)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>SVG generation</td> <td class="desc">SVG Arena benchmark for icon generation evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="Intern-S1" data-provider="internlm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=intern-s1-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Intern-S1-Pro</span> </a> </td> <td data-sort="83.5"> 83.5% </td> </tr> <tr data-bench="scale-ai-multi-challenge" data-open-source="1"> <td style="text-align:center" data-sort="545">545</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=scale-ai-multi-challenge" class="llm-fu-bench-link" data-bench-id="scale-ai-multi-challenge" title="View models and scores"><strong>Scale AI Multi Challenge</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Chat &amp; instruction following</td> <td class="desc">Scale AI Multi Challenge crowd-evaluated instruction following benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-30b-a3b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-30B-A3B-Thinking-2507</span> </a> </td> <td data-sort="44.8"> 44.8% </td> </tr> <tr data-bench="scicode" data-open-source="0"> <td style="text-align:center" data-sort="546">546</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=scicode" class="llm-fu-bench-link" data-bench-id="scicode" title="View models and scores"><strong>SciCode (sub)</strong></a> <span class="score-symbol" title="78 scores" aria-label="78 scores">O</span> </td> <td>Code</td> <td class="desc">SciCode subset score (sub).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="59.0"> 59.0% </td> </tr> <tr data-bench="scicode-main" data-open-source="0"> <td style="text-align:center" data-sort="547">547</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=scicode-main" class="llm-fu-bench-link" data-bench-id="scicode-main" title="View models and scores"><strong>SciCode (main)</strong></a> <span class="score-symbol" title="23 scores" aria-label="23 scores">O</span> </td> <td>Code</td> <td class="desc">SciCode main score.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="15.4"> 15.4% </td> </tr> <tr data-bench="scienceqa" data-open-source="1"> <td style="text-align:center" data-sort="548">548</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=scienceqa" class="llm-fu-bench-link" data-bench-id="scienceqa" title="View models and scores"><strong>ScienceQA</strong></a> <span class="score-symbol" title="30 scores" aria-label="30 scores">O</span> </td> <td>Science QA (multimodal)</td> <td class="desc">Multiple-choice science questions with images, diagrams, and text context.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="FastVLM" data-model-version="FastVLM" data-provider="apple"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=fastvlm-7b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">FastVLM-7B</span> </a> </td> <td data-sort="96.7"> 96.7% </td> </tr> <tr data-bench="sciq" data-open-source="1"> <td style="text-align:center" data-sort="549">549</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=sciq" class="llm-fu-bench-link" data-bench-id="sciq" title="View models and scores"><strong>SciQ</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://allenai.org/data/sciq" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Science QA</td> <td class="desc">Multiple choice science questions.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Pythia" data-model-version="Pythia" data-provider="eleutherai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=pythia-12b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Pythia 12B</span> </a> </td> <td data-sort="92.9"> 92.9% </td> </tr> <tr data-bench="scireasoner" data-open-source="1"> <td style="text-align:center" data-sort="550">550</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=scireasoner" class="llm-fu-bench-link" data-bench-id="scireasoner" title="View models and scores"><strong>SciReasoner</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Scientific reasoning</td> <td class="desc">Scientific reasoning benchmark evaluating multimodal AI models on scientific tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="Intern-S1" data-provider="internlm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=intern-s1-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Intern-S1-Pro</span> </a> </td> <td data-sort="55.5"> 55.5% </td> </tr> <tr data-bench="scires-frontiermath-tier-1-3" data-open-source="0"> <td style="text-align:center" data-sort="551">551</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=scires-frontiermath-tier-1-3" class="llm-fu-bench-link" data-bench-id="scires-frontiermath-tier-1-3" title="View models and scores"><strong>SciRes FrontierMath Tier 1-3</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Math (frontier)</td> <td class="desc">SciRes FrontierMath benchmark covering tiers 1-3.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-thinking"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Thinking</span> </a> </td> <td data-sort="40.3"> 40.3% </td> </tr> <tr data-bench="scires-frontiermath-tier-4" data-open-source="0"> <td style="text-align:center" data-sort="552">552</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=scires-frontiermath-tier-4" class="llm-fu-bench-link" data-bench-id="scires-frontiermath-tier-4" title="View models and scores"><strong>SciRes FrontierMath Tier 4</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Math (frontier)</td> <td class="desc">SciRes FrontierMath benchmark covering tier 4.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="18.8"> 18.8% </td> </tr> <tr data-bench="screenqa-complex" data-open-source="1"> <td style="text-align:center" data-sort="553">553</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=screenqa-complex" class="llm-fu-bench-link" data-bench-id="screenqa-complex" title="View models and scores"><strong>ScreenQA Complex</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>GUI QA</td> <td class="desc">Complex ScreenQA benchmark accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Holo" data-model-version="Holo1.5" data-provider="hcompany"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=holo1.5-72b"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Holo1.5-72B</span> </a> </td> <td data-sort="87.1"> 87.1% </td> </tr> <tr data-bench="screenqa-short" data-open-source="1"> <td style="text-align:center" data-sort="554">554</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=screenqa-short" class="llm-fu-bench-link" data-bench-id="screenqa-short" title="View models and scores"><strong>ScreenQA Short</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>GUI QA</td> <td class="desc">Short-form ScreenQA benchmark accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Holo" data-model-version="Holo1.5" data-provider="hcompany"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=holo1.5-72b"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Holo1.5-72B</span> </a> </td> <td data-sort="91.9"> 91.9% </td> </tr> <tr data-bench="screenspot" data-open-source="1"> <td style="text-align:center" data-sort="555">555</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=screenspot" class="llm-fu-bench-link" data-bench-id="screenspot" title="View models and scores"><strong>ScreenSpot</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> </td> <td>Screen UI locators</td> <td class="desc">Center accuracy on ScreenSpot.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-32b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL 32B Instruct</span> </a> </td> <td data-sort="95.8"> 95.8% </td> </tr> <tr data-bench="screenspot-pro" data-open-source="0"> <td style="text-align:center" data-sort="556">556</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=screenspot-pro" class="llm-fu-bench-link" data-bench-id="screenspot-pro" title="View models and scores"><strong>ScreenSpot-Pro</strong></a> <span class="score-symbol" title="55 scores" aria-label="55 scores">O</span> <a href="https://gui-agent.github.io/grounding-leaderboard/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Screen UI locators</td> <td class="desc">Average center accuracy on ScreenSpot-Pro.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-extra-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Extra High</span> </a> </td> <td data-sort="86.3"> 86.3% </td> </tr> <tr data-bench="screenspot-v2" data-open-source="1"> <td style="text-align:center" data-sort="557">557</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=screenspot-v2" class="llm-fu-bench-link" data-bench-id="screenspot-v2" title="View models and scores"><strong>ScreenSpot-v2</strong></a> <span class="score-symbol" title="25 scores" aria-label="25 scores">O</span> </td> <td>Screen UI locators</td> <td class="desc">Center accuracy on ScreenSpot-v2.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="UI-Venus" data-model-version="UI-Venus" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ui-venus-72b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">UI-Venus 72B</span> </a> </td> <td data-sort="95.3"> 95.3% </td> </tr> <tr data-bench="seal-0" data-open-source="1"> <td style="text-align:center" data-sort="558">558</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=seal-0" class="llm-fu-bench-link" data-bench-id="seal-0" title="View models and scores"><strong>SEAL-0</strong></a> <span class="score-symbol" title="24 scores" aria-label="24 scores">O</span> </td> <td>Agentic web search</td> <td class="desc">Evaluation of multi-step browsing agents on search, evidence gathering, and synthesis tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5 Thinking</span> </a> </td> <td data-sort="57.4"> 57.4% </td> </tr> <tr data-bench="seccodebench" data-open-source="0"> <td style="text-align:center" data-sort="559">559</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=seccodebench" class="llm-fu-bench-link" data-bench-id="seccodebench" title="View models and scores"><strong>SecCodeBench</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Secure code generation</td> <td class="desc">Benchmark evaluating secure code generation capabilities.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="68.7"> 68.7% </td> </tr> <tr data-bench="seed-bench-2-plus" data-open-source="0"> <td style="text-align:center" data-sort="560">560</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=seed-bench-2-plus" class="llm-fu-bench-link" data-bench-id="seed-bench-2-plus" title="View models and scores"><strong>SEED-Bench-2-Plus</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> <a href="https://huggingface.co/datasets/AILab-CVC/SEED-Bench-2-plus" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2404.16790" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Multimodal evaluation</td> <td class="desc">SEED-Bench-2-Plus overall accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 3.7" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-3.7-sonnet"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude 3.7 Sonnet</span> </a> </td> <td data-sort="72.9"> 72.9% </td> </tr> <tr data-bench="seed-bench-img" data-open-source="1"> <td style="text-align:center" data-sort="561">561</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=seed-bench-img" class="llm-fu-bench-link" data-bench-id="seed-bench-img" title="View models and scores"><strong>SEED-Bench-Img</strong></a> <span class="score-symbol" title="31 scores" aria-label="31 scores">O</span> </td> <td>Multimodal image understanding</td> <td class="desc">SEED-Bench image-only subset (SEED-Bench-Img).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Bagel" data-model-version="Bagel" data-provider="jondurbin"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=bagel-14b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Bagel 14B</span> </a> </td> <td data-sort="78.5"> 78.5% </td> </tr> <tr data-bench="seedbench" data-open-source="1"> <td style="text-align:center" data-sort="562">562</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=seedbench" class="llm-fu-bench-link" data-bench-id="seedbench" title="View models and scores"><strong>SEED-Bench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://huggingface.co/datasets/AILab-CVC/SEED-Bench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2307.16125" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/AILab-CVC/SEED-Bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multimodal evaluation</td> <td class="desc">SEED-Bench comprehensive multimodal understanding benchmark evaluating generative comprehension across multiple dimensions.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LFM" data-model-version="LFM2-VL" data-provider="liquidai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=lfm2-vl-3b"> <span class="brand"> <img class="logo no-invert" alt="liquidai logo" data-company="liquidai" data-country="United States" src="https://furukama.com/logos/liquidai.svg?v=817b8d2c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">LFM2-VL-3B</span> </a> </td> <td data-sort="76.5"> 76.5% </td> </tr> <tr data-bench="sfe" data-open-source="0"> <td style="text-align:center" data-sort="563">563</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=sfe" class="llm-fu-bench-link" data-bench-id="sfe" title="View models and scores"><strong>SFE</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Multimodal reasoning</td> <td class="desc">Structured factual evaluation for multimodal models.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="61.9"> 61.9% </td> </tr> <tr data-bench="showdown" data-open-source="1"> <td style="text-align:center" data-sort="564">564</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=showdown" class="llm-fu-bench-link" data-bench-id="showdown" title="View models and scores"><strong>Showdown</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>GUI agents</td> <td class="desc">Success rate on the Showdown UI interaction benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Holo" data-model-version="Holo1.5" data-provider="hcompany"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=holo1.5-72b"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Holo1.5-72B</span> </a> </td> <td data-sort="76.8"> 76.8% </td> </tr> <tr data-bench="sifo" data-open-source="1"> <td style="text-align:center" data-sort="565">565</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=sifo" class="llm-fu-bench-link" data-bench-id="sifo" title="View models and scores"><strong>SIFO</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Instruction following</td> <td class="desc">Single-turn instruction following benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-30b-a3b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-30B-A3B Thinking</span> </a> </td> <td data-sort="66.9"> 66.9% </td> </tr> <tr data-bench="sifo-multiturn" data-open-source="1"> <td style="text-align:center" data-sort="566">566</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=sifo-multiturn" class="llm-fu-bench-link" data-bench-id="sifo-multiturn" title="View models and scores"><strong>SIFO Multiturn</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Instruction following</td> <td class="desc">Multi-turn SIFO benchmark for sustained instruction adherence.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-30b-a3b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-30B-A3B Thinking</span> </a> </td> <td data-sort="60.3"> 60.3% </td> </tr> <tr data-bench="simpleqa" data-open-source="1"> <td style="text-align:center" data-sort="567">567</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=simpleqa" class="llm-fu-bench-link" data-bench-id="simpleqa" title="View models and scores"><strong>SimpleQA</strong></a> <span class="score-symbol" title="121 scores" aria-label="121 scores">O</span> </td> <td>QA</td> <td class="desc">Simple question answering benchmark.</td> <td data-sort="2"> <span class="stars" title="Usage: 8/39; GitHub>100: no; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="DeepSeek" data-model-version="DeepSeek V3.2" data-provider="deepseek"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=deepseek-v3.2-exp"> <span class="brand"> <img class="logo" alt="deepseek logo" data-company="deepseek" data-country="China" src="https://furukama.com/logos/deepseek.svg?v=69c5b837" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">DeepSeek V3.2-Exp</span> </a> </td> <td data-sort="97.1"> 97.1% </td> </tr> <tr data-bench="simpleqa-verified" data-open-source="0"> <td style="text-align:center" data-sort="568">568</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=simpleqa-verified" class="llm-fu-bench-link" data-bench-id="simpleqa-verified" title="View models and scores"><strong>SimpleQA Verified</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> </td> <td>QA</td> <td class="desc">Verified SimpleQA variant for parametric knowledge accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="72.1"> 72.1% </td> </tr> <tr data-bench="simplevqa" data-open-source="0"> <td style="text-align:center" data-sort="569">569</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=simplevqa" class="llm-fu-bench-link" data-bench-id="simplevqa" title="View models and scores"><strong>SimpleVQA</strong></a> <span class="score-symbol" title="37 scores" aria-label="37 scores">O</span> </td> <td>General VQA</td> <td class="desc">Lightweight visual question answering set with everyday scenes.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="73.2"> 73.2% </td> </tr> <tr data-bench="simplevqa-ds" data-open-source="0"> <td style="text-align:center" data-sort="570">570</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=simplevqa-ds" class="llm-fu-bench-link" data-bench-id="simplevqa-ds" title="View models and scores"><strong>SimpleVQA-DS</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>General VQA</td> <td class="desc">SimpleVQA variant curated by DeepSeek with everyday image question answering tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.5" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.5-vl-thinking"> <span class="brand"> <img class="logo no-invert" alt="bytedance logo" data-company="bytedance" data-country="China" src="https://furukama.com/logos/bytedance.svg?v=4e7ed8af" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Seed1.5-VL-Thinking</span> </a> </td> <td data-sort="61.3"> 61.3% </td> </tr> <tr data-bench="siqa" data-open-source="1"> <td style="text-align:center" data-sort="571">571</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=siqa" class="llm-fu-bench-link" data-bench-id="siqa" title="View models and scores"><strong>Social Interaction QA (SIQA)</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Social commonsense QA</td> <td class="desc">Social Interaction QA benchmark evaluating social commonsense and situational reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemma" data-model-version="Gemma 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemma-3-27b"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemma 3 27B</span> </a> </td> <td data-sort="54.9"> 54.9% </td> </tr> <tr data-bench="slake" data-open-source="1"> <td style="text-align:center" data-sort="572">572</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=slake" class="llm-fu-bench-link" data-bench-id="slake" title="View models and scores"><strong>SLAKE</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Medical VQA</td> <td class="desc">Semantically-Labeled Knowledge-Enhanced medical visual question answering benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Kimi" data-model-version="Kimi K2.5" data-provider="moonshotai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=kimi-k2.5"> <span class="brand"> <img class="logo no-invert" alt="kimi logo" data-company="kimi" data-country="China" src="https://furukama.com/logos/kimi.svg?v=87618e90" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Kimi-K2.5</span> </a> </td> <td data-sort="81.6"> 81.6% </td> </tr> <tr data-bench="smolinstruct" data-open-source="1"> <td style="text-align:center" data-sort="573">573</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=smolinstruct" class="llm-fu-bench-link" data-bench-id="smolinstruct" title="View models and scores"><strong>SmolInstruct</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Small molecule understanding</td> <td class="desc">Small molecule instruction-following and understanding benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="Intern-S1" data-provider="internlm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=intern-s1-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Intern-S1-Pro</span> </a> </td> <td data-sort="74.8"> 74.8% </td> </tr> <tr data-bench="socialiqa" data-open-source="1"> <td style="text-align:center" data-sort="574">574</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=socialiqa" class="llm-fu-bench-link" data-bench-id="socialiqa" title="View models and scores"><strong>SocialIQA</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://leaderboard.allenai.org/socialiqa" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Social commonsense</td> <td class="desc">Social interaction commonsense QA.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemma" data-model-version="Gemma 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemma-3-pt-27b"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemma 3 PT 27B</span> </a> </td> <td data-sort="54.9"> 54.9% </td> </tr> <tr data-bench="spatialviz" data-open-source="0"> <td style="text-align:center" data-sort="575">575</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=spatialviz" class="llm-fu-bench-link" data-bench-id="spatialviz" title="View models and scores"><strong>SpatialViz</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Mental visualization</td> <td class="desc">Mental visualization benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="65.8"> 65.8% </td> </tr> <tr data-bench="spider" data-open-source="1"> <td style="text-align:center" data-sort="576">576</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=spider" class="llm-fu-bench-link" data-bench-id="spider" title="View models and scores"><strong>Spider</strong></a> <span class="score-symbol" title="21 scores" aria-label="21 scores">O</span> <a href="https://yale-lily.github.io/spider" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Text-to-SQL</td> <td class="desc">Complex text-to-SQL benchmark over cross-domain databases.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LLaDA" data-model-version="LLaDA 2.0" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llada2.0-flash"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">LLaDA2.0 Flash</span> </a> </td> <td data-sort="82.5"> 82.5% </td> </tr> <tr data-bench="spiral-bench" data-open-source="0"> <td style="text-align:center" data-sort="577">577</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=spiral-bench" class="llm-fu-bench-link" data-bench-id="spiral-bench" title="View models and scores"><strong>Spiral-Bench</strong></a> <span class="score-symbol" title="10 scores" aria-label="10 scores">O</span> <a href="https://eqbench.com/spiral-bench.html" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://eqbench.com/spiral-bench.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2312.06281" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/EQ-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Safety / sycophancy</td> <td class="desc">A LLM-judged benchmark measuring sycophancy and delusion reinforcement.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="87.0"> 87.0% </td> </tr> <tr data-bench="squad-v1.1" data-open-source="1"> <td style="text-align:center" data-sort="578">578</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=squad-v1.1" class="llm-fu-bench-link" data-bench-id="squad-v1.1" title="View models and scores"><strong>SQuAD v1.1</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://github.com/rajpurkar/SQuAD-explorer" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Reading comprehension</td> <td class="desc">Extractive QA from Wikipedia articles.</td> <td data-sort="2"> <span class="stars" title="Usage: 1/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="566">566</td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b-base"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B Base</span> </a> </td> <td data-sort="89.3"> 89.3% </td> </tr> <tr data-bench="squad-v2.0" data-open-source="1"> <td style="text-align:center" data-sort="579">579</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=squad-v2.0" class="llm-fu-bench-link" data-bench-id="squad-v2.0" title="View models and scores"><strong>SQuAD v2.0</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> <a href="https://github.com/rajpurkar/SQuAD-explorer" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Reading comprehension</td> <td class="desc">Like v1.1 with unanswerable questions.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="566">566</td> <td class="leader-cell" data-model-family="LLaDA" data-model-version="LLaDA 2.1" data-provider="antgroup"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llada2.1-flash-q"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">LLaDA2.1-Flash (Q Mode)</span> </a> </td> <td data-sort="90.8"> 90.8% </td> </tr> <tr data-bench="streamingbench" data-open-source="0"> <td style="text-align:center" data-sort="580">580</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=streamingbench" class="llm-fu-bench-link" data-bench-id="streamingbench" title="View models and scores"><strong>StreamingBench</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Streaming video</td> <td class="desc">Streaming video understanding benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="84.4"> 84.4% </td> </tr> <tr data-bench="sunrgbd" data-open-source="0"> <td style="text-align:center" data-sort="581">581</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=sunrgbd" class="llm-fu-bench-link" data-bench-id="sunrgbd" title="View models and scores"><strong>SUNRGBD</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>3D scene understanding</td> <td class="desc">SUN RGB-D benchmark for indoor scene understanding from RGB-D imagery.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-mini-minimal"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 Mini Minimal</span> </a> </td> <td data-sort="45.8"> 45.8% </td> </tr> <tr data-bench="superchem" data-open-source="0"> <td style="text-align:center" data-sort="582">582</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=superchem" class="llm-fu-bench-link" data-bench-id="superchem" title="View models and scores"><strong>SuperChem</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Chemistry reasoning</td> <td class="desc">Chemistry reasoning benchmark evaluating text-based chemical knowledge and problem solving.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="63.2"> 63.2% </td> </tr> <tr data-bench="supergpqa" data-open-source="0"> <td style="text-align:center" data-sort="583">583</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=supergpqa" class="llm-fu-bench-link" data-bench-id="supergpqa" title="View models and scores"><strong>SuperGPQA</strong></a> <span class="score-symbol" title="93 scores" aria-label="93 scores">O</span> </td> <td>Graduate-level QA</td> <td class="desc">Harder GPQA variant assessing advanced graduate-level reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="75.3"> 75.3% </td> </tr> <tr data-bench="swe-bench" data-open-source="0"> <td style="text-align:center" data-sort="584">584</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-bench" class="llm-fu-bench-link" data-bench-id="swe-bench" title="View models and scores"><strong>SWE-Bench</strong></a> <span class="score-symbol" title="32 scores" aria-label="32 scores">O</span> <a href="https://github.com/princeton-nlp/SWE-bench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code repair</td> <td class="desc">Supervised software engineering benchmark across many repos and issues.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="3442">3442</td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-codex"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 Codex</span> </a> </td> <td data-sort="74.5"> 74.5% </td> </tr> <tr data-bench="swe-bench-multilingual" data-open-source="0"> <td style="text-align:center" data-sort="585">585</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-bench-multilingual" class="llm-fu-bench-link" data-bench-id="swe-bench-multilingual" title="View models and scores"><strong>SWE-Bench Multilingual</strong></a> <span class="score-symbol" title="52 scores" aria-label="52 scores">O</span> </td> <td>Code repair (multilingual)</td> <td class="desc">Multilingual variant of SWE-Bench for issue fixing.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5 Thinking</span> </a> </td> <td data-sort="77.5"> 77.5% </td> </tr> <tr data-bench="swe-bench-openhands" data-open-source="1"> <td style="text-align:center" data-sort="586">586</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-bench-openhands" class="llm-fu-bench-link" data-bench-id="swe-bench-openhands" title="View models and scores"><strong>SWE-Bench (OpenHands)</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://github.com/princeton-nlp/SWE-bench" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code repair</td> <td class="desc">SWE-Bench results using the OpenHands autonomous coding agent.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="3442">3442</td> <td class="leader-cell" data-model-family="Nemotron" data-model-version="Nemotron 3" data-provider="nvidia"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nvidia-nemotron-3-nano-30b-a3b-bf16"> <span class="brand"> <img class="logo no-invert" alt="nvidia logo" data-company="nvidia" data-country="United States" src="https://furukama.com/logos/nvidia.svg?v=3b09dc1a" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">NVIDIA-Nemotron-3-Nano-30B-A3B-BF16</span> </a> </td> <td data-sort="38.8"> 38.8% </td> </tr> <tr data-bench="swe-bench-pro" data-open-source="0"> <td style="text-align:center" data-sort="587">587</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-bench-pro" class="llm-fu-bench-link" data-bench-id="swe-bench-pro" title="View models and scores"><strong>SWE-Bench Pro</strong></a> <span class="score-symbol" title="20 scores" aria-label="20 scores">O</span> <a href="https://www.swebench.com/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Software engineering</td> <td class="desc">Full SWE-Bench Pro benchmark for software-engineering agents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="56.9"> 56.9% </td> </tr> <tr data-bench="swe-bench-pro-public" data-open-source="0"> <td style="text-align:center" data-sort="588">588</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-bench-pro-public" class="llm-fu-bench-link" data-bench-id="swe-bench-pro-public" title="View models and scores"><strong>SWE-Bench Pro (Public)</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Software engineering</td> <td class="desc">Public subset of the SWE-Bench Pro benchmark for software-engineering agents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.3 Codex" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.3-codex"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.3 Codex</span> </a> </td> <td data-sort="56.8"> 56.8% </td> </tr> <tr data-bench="swe-bench-verified" data-open-source="0"> <td style="text-align:center" data-sort="589">589</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-bench-verified" class="llm-fu-bench-link" data-bench-id="swe-bench-verified" title="View models and scores"><strong>SWE-Bench Verified</strong></a> <span class="score-symbol" title="192 scores" aria-label="192 scores">O</span> <a href="https://www.swebench.com/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Code repair</td> <td class="desc">Verified subset of SWE-Bench for issue fixing.</td> <td data-sort="3"> <span class="stars" title="Usage: 12/39; GitHub>100: no; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="80.9"> 80.9% </td> </tr> <tr data-bench="swe-dev" data-open-source="0"> <td style="text-align:center" data-sort="590">590</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-dev" class="llm-fu-bench-link" data-bench-id="swe-dev" title="View models and scores"><strong>SWE-Dev</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Code repair</td> <td class="desc">Software engineering development and bug fixing benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4</span> </a> </td> <td data-sort="67.1"> 67.1% </td> </tr> <tr data-bench="swe-lancer" data-open-source="0"> <td style="text-align:center" data-sort="591">591</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-lancer" class="llm-fu-bench-link" data-bench-id="swe-lancer" title="View models and scores"><strong>SWE-Lancer</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://arxiv.org/abs/2502.12115" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Code repair (freelance tasks)</td> <td class="desc">Software engineering benchmark using real freelance-style issues.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.1 Codex" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.1-codex-max"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.1 Codex-Max</span> </a> </td> <td data-sort="79.9"> 79.9% </td> </tr> <tr data-bench="swe-lancer-diamond" data-open-source="0"> <td style="text-align:center" data-sort="592">592</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-lancer-diamond" class="llm-fu-bench-link" data-bench-id="swe-lancer-diamond" title="View models and scores"><strong>SWE-Lancer Diamond</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Code repair (freelance)</td> <td class="desc">Diamond subset of SWE-Lancer focusing on the hardest freelance-style issues.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.3 Codex" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.3-codex"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.3 Codex</span> </a> </td> <td data-sort="81.4"> 81.4% </td> </tr> <tr data-bench="swe-lancer-ic-diamond" data-open-source="0"> <td style="text-align:center" data-sort="593">593</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-lancer-ic-diamond" class="llm-fu-bench-link" data-bench-id="swe-lancer-ic-diamond" title="View models and scores"><strong>SWE-Lancer IC Diamond</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Code repair (freelance)</td> <td class="desc">Individual Contributor Diamond subset of SWE-Lancer.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.3 Codex" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.3-codex"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.3 Codex</span> </a> </td> <td data-sort="81.4"> 81.4% </td> </tr> <tr data-bench="swe-perf" data-open-source="0"> <td style="text-align:center" data-sort="594">594</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-perf" class="llm-fu-bench-link" data-bench-id="swe-perf" title="View models and scores"><strong>SWE-Perf</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Code repair</td> <td class="desc">Software engineering benchmark focused on performance-oriented fixes.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="6.5"> 6.5% </td> </tr> <tr data-bench="swe-review" data-open-source="0"> <td style="text-align:center" data-sort="595">595</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swe-review" class="llm-fu-bench-link" data-bench-id="swe-review" title="View models and scores"><strong>SWE-Review</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Code review</td> <td class="desc">Software engineering review benchmark for assessing code review quality.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="16.2"> 16.2% </td> </tr> <tr data-bench="swt-bench" data-open-source="0"> <td style="text-align:center" data-sort="596">596</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=swt-bench" class="llm-fu-bench-link" data-bench-id="swt-bench" title="View models and scores"><strong>SWT-Bench</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Code repair</td> <td class="desc">Software tool-use benchmark for code tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2-thinking"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2 Thinking</span> </a> </td> <td data-sort="80.7"> 80.7% </td> </tr> <tr data-bench="sysbench" data-open-source="0"> <td style="text-align:center" data-sort="597">597</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=sysbench" class="llm-fu-bench-link" data-bench-id="sysbench" title="View models and scores"><strong>SysBench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>System prompts</td> <td class="desc">System prompt understanding and adherence benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4.1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4.1"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4.1</span> </a> </td> <td data-sort="74.1"> 74.1% </td> </tr> <tr data-bench="tau1-airline" data-open-source="1"> <td style="text-align:center" data-sort="598">598</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tau1-airline" class="llm-fu-bench-link" data-bench-id="tau1-airline" title="View models and scores"><strong>TAU1-Airline</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Agent tasks (airline)</td> <td class="desc">Tool-augmented agent evaluation in airline scenarios (TAU1).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="openPangu-R" data-model-version="openPangu-R" data-provider="freedomintelligence"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=openpangu-r-72b-2512-slow-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">openPangu-R-72B-2512 Slow Thinking</span> </a> </td> <td data-sort="56.0"> 56.0% </td> </tr> <tr data-bench="tau1-retail" data-open-source="1"> <td style="text-align:center" data-sort="599">599</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tau1-retail" class="llm-fu-bench-link" data-bench-id="tau1-retail" title="View models and scores"><strong>TAU1-Retail</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Agent tasks (retail)</td> <td class="desc">Tool-augmented agent evaluation in retail scenarios (TAU1).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="openPangu-R" data-model-version="openPangu-R" data-provider="freedomintelligence"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=openpangu-r-72b-2512-slow-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">openPangu-R-72B-2512 Slow Thinking</span> </a> </td> <td data-sort="73.0"> 73.0% </td> </tr> <tr data-bench="tau2-airline" data-open-source="1"> <td style="text-align:center" data-sort="600">600</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tau2-airline" class="llm-fu-bench-link" data-bench-id="tau2-airline" title="View models and scores"><strong>TAU2-Airline</strong></a> <span class="score-symbol" title="52 scores" aria-label="52 scores">O</span> </td> <td>Agent tasks (airline)</td> <td class="desc">Tool-augmented agent evaluation in airline scenarios (TAU2).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LongCat" data-model-version="LongCat-Flash" data-provider="meituan"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=longcat-flash-thinking-2601"> <span class="brand"> <img class="logo no-invert" alt="meituan logo" data-company="meituan" data-country="China" src="https://furukama.com/logos/meituan.svg?v=45908c84" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">LongCat-Flash-Thinking-2601</span> </a> </td> <td data-sort="76.5"> 76.5% </td> </tr> <tr data-bench="tau2-bench" data-open-source="0"> <td style="text-align:center" data-sort="601">601</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tau2-bench" class="llm-fu-bench-link" data-bench-id="tau2-bench" title="View models and scores"><strong>TAU2-Bench</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Agent tasks</td> <td class="desc">Aggregate tool-augmented agent evaluation across airline, retail, and telecom scenarios (TAU2).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="91.6"> 91.6% </td> </tr> <tr data-bench="tau2-retail" data-open-source="0"> <td style="text-align:center" data-sort="602">602</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tau2-retail" class="llm-fu-bench-link" data-bench-id="tau2-retail" title="View models and scores"><strong>TAU2-Retail</strong></a> <span class="score-symbol" title="66 scores" aria-label="66 scores">O</span> </td> <td>Agent tasks (retail)</td> <td class="desc">Tool-augmented agent evaluation in retail scenarios (TAU2).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.6" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.6"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.6</span> </a> </td> <td data-sort="91.9"> 91.9% </td> </tr> <tr data-bench="tau2-telecom" data-open-source="1"> <td style="text-align:center" data-sort="603">603</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tau2-telecom" class="llm-fu-bench-link" data-bench-id="tau2-telecom" title="View models and scores"><strong>TAU2-Telecom</strong></a> <span class="score-symbol" title="136 scores" aria-label="136 scores">O</span> </td> <td>Agent tasks (telecom)</td> <td class="desc">Tool-augmented agent evaluation in telecom scenarios (TAU2).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="LongCat" data-model-version="LongCat-Flash" data-provider="meituan"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=longcat-flash-thinking-2601"> <span class="brand"> <img class="logo no-invert" alt="meituan logo" data-company="meituan" data-country="China" src="https://furukama.com/logos/meituan.svg?v=45908c84" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">LongCat-Flash-Thinking-2601</span> </a> </td> <td data-sort="99.3"> 99.3% </td> </tr> <tr data-bench="tempcompass" data-open-source="0"> <td style="text-align:center" data-sort="604">604</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tempcompass" class="llm-fu-bench-link" data-bench-id="tempcompass" title="View models and scores"><strong>TempCompass</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Temporal reasoning</td> <td class="desc">Temporal reasoning benchmark evaluating understanding of time-related concepts in videos and images.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="88.0"> 88.0% </td> </tr> <tr data-bench="terminal-bench" data-open-source="0"> <td style="text-align:center" data-sort="605">605</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=terminal-bench" class="llm-fu-bench-link" data-bench-id="terminal-bench" title="View models and scores"><strong>Terminal-Bench</strong></a> <span class="score-symbol" title="109 scores" aria-label="109 scores">O</span> <a href="https://www.tbench.ai/leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://www.tbench.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/laude-institute/terminal-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Agent terminal tasks</td> <td class="desc">Command-line task completion benchmark for agents.</td> <td data-sort="3"> <span class="stars" title="Usage: 6/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★★<span class="empty">★★</span> </span> </td> <td data-sort="637">637</td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-4.5-sonnet-thinking"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5 (Thinking)</span> </a> </td> <td data-sort="61.3"> 61.3% </td> </tr> <tr data-bench="terminal-bench-2.0" data-open-source="1"> <td style="text-align:center" data-sort="606">606</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=terminal-bench-2.0" class="llm-fu-bench-link" data-bench-id="terminal-bench-2.0" title="View models and scores"><strong>Terminal-Bench 2.0</strong></a> <span class="score-symbol" title="94 scores" aria-label="94 scores">O</span> <a href="https://www.tbench.ai/leaderboard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://www.tbench.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/laude-institute/terminal-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Agent terminal tasks</td> <td class="desc">Second-generation Terminal-Bench leaderboard for end-to-end terminal agents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-loop-instruct"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Loop-Instruct</span> </a> </td> <td data-sort="81.4"> 81.4% </td> </tr> <tr data-bench="terminal-bench-hard" data-open-source="0"> <td style="text-align:center" data-sort="607">607</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=terminal-bench-hard" class="llm-fu-bench-link" data-bench-id="terminal-bench-hard" title="View models and scores"><strong>Terminal-Bench Hard</strong></a> <span class="score-symbol" title="70 scores" aria-label="70 scores">O</span> <a href="https://artificialanalysis.ai/evaluations/terminalbench-hard" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://www.tbench.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Agent terminal tasks</td> <td class="desc">Hard subset of Terminal-Bench command-line agent tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.1" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.1-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.1 High</span> </a> </td> <td data-sort="43.0"> 43.0% </td> </tr> <tr data-bench="terminal-bench-terminus" data-open-source="0"> <td style="text-align:center" data-sort="608">608</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=terminal-bench-terminus" class="llm-fu-bench-link" data-bench-id="terminal-bench-terminus" title="View models and scores"><strong>Terminal-Bench Terminus</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> <a href="https://www.tbench.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/laude-institute/terminal-bench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Agent terminal tasks</td> <td class="desc">Terminal-Bench Terminus track assessing end-to-end terminal tool use.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3.1" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3.1-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3.1 Pro</span> </a> </td> <td data-sort="68.5"> 68.5% </td> </tr> <tr data-bench="textquests" data-open-source="0"> <td style="text-align:center" data-sort="609">609</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=textquests" class="llm-fu-bench-link" data-bench-id="textquests" title="View models and scores"><strong>TextQuests</strong></a> <span class="score-symbol" title="18 scores" aria-label="18 scores">O</span> </td> <td>Text-based video games</td> <td class="desc">Text-based video game benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="41.0"> 41.0% </td> </tr> <tr data-bench="textquests-harm" data-open-source="0"> <td style="text-align:center" data-sort="610">610</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=textquests-harm" class="llm-fu-bench-link" data-bench-id="textquests-harm" title="View models and scores"><strong>TextQuests Harm</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Harmful propensities</td> <td class="desc">Harmfulness evaluation on TextQuests scenarios.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Grok" data-model-version="Grok 4.1" data-provider="xai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=grok-4.1-fast"> <span class="brand"> <img class="logo no-invert" alt="xai logo" data-company="xai" data-country="United States" src="https://furukama.com/logos/xai.svg?v=89da333c" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Grok 4.1 Fast</span> </a> </td> <td data-sort="9.1"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 9.1%</span> </td> </tr> <tr data-bench="textvqa" data-open-source="1"> <td style="text-align:center" data-sort="611">611</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=textvqa" class="llm-fu-bench-link" data-bench-id="textvqa" title="View models and scores"><strong>TextVQA</strong></a> <span class="score-symbol" title="82 scores" aria-label="82 scores">O</span> <a href="https://textvqa.org/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Text-based VQA</td> <td class="desc">Visual question answering that requires reading text in images.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Ovis" data-model-version="Ovis 2.6" data-provider="alibaba"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ovis2.6-30b-a3b"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Ovis2.6-30B-A3B</span> </a> </td> <td data-sort="90.7"> 90.7% </td> </tr> <tr data-bench="tiif-bench-long" data-open-source="0"> <td style="text-align:center" data-sort="612">612</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tiif-bench-long" class="llm-fu-bench-link" data-bench-id="tiif-bench-long" title="View models and scores"><strong>TIIF-Bench Long</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Text-to-image</td> <td class="desc">TIIF-Bench long prompt score for text-to-image generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seedream" data-model-version="Seedream" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seedream-4.5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seedream 4.5</span> </a> </td> <td data-sort="88.5"> 88.5% </td> </tr> <tr data-bench="tiif-bench-short" data-open-source="0"> <td style="text-align:center" data-sort="613">613</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tiif-bench-short" class="llm-fu-bench-link" data-bench-id="tiif-bench-short" title="View models and scores"><strong>TIIF-Bench Short</strong></a> <span class="score-symbol" title="16 scores" aria-label="16 scores">O</span> </td> <td>Text-to-image</td> <td class="desc">TIIF-Bench short prompt score for text-to-image generation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Nano Banana" data-model-version="Nano Banana" data-provider="unknown"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=nano-banana-2.0"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Nano Banana 2.0</span> </a> </td> <td data-sort="91.0"> 91.0% </td> </tr> <tr data-bench="tir-bench" data-open-source="1"> <td style="text-align:center" data-sort="614">614</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tir-bench" class="llm-fu-bench-link" data-bench-id="tir-bench" title="View models and scores"><strong>TIR-Bench</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Tool-integrated reasoning</td> <td class="desc">Benchmark for tool-integrated reasoning with visual models.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-27b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-27B</span> </a> </td> <td data-sort="59.8"> 59.8% </td> </tr> <tr data-bench="tldr9-plus" data-open-source="1"> <td style="text-align:center" data-sort="615">615</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tldr9-plus" class="llm-fu-bench-link" data-bench-id="tldr9-plus" title="View models and scores"><strong>TLDR9&#43;</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Summarization</td> <td class="desc">Long-form summarization benchmark with nine-domain TLDR prompts plus extended variations.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MobileLLM" data-model-version="MobileLLM" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mobilellm-p1"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MobileLLM P1</span> </a> </td> <td data-sort="16.8"> 16.8% </td> </tr> <tr data-bench="tomato" data-open-source="0"> <td style="text-align:center" data-sort="616">616</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tomato" class="llm-fu-bench-link" data-bench-id="tomato" title="View models and scores"><strong>TOMATO</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Temporal understanding</td> <td class="desc">Temporal ordering and motion analysis benchmark (TOMATO).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="60.8"> 60.8% </td> </tr> <tr data-bench="tool-decathlon" data-open-source="0"> <td style="text-align:center" data-sort="617">617</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tool-decathlon" class="llm-fu-bench-link" data-bench-id="tool-decathlon" title="View models and scores"><strong>Tool-Decathlon</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Agent tool-use</td> <td class="desc">Composite tool-use suite measuring multi-domain tool invocation success (Pass@1).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="43.8"> 43.8% </td> </tr> <tr data-bench="toolathlon" data-open-source="0"> <td style="text-align:center" data-sort="618">618</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=toolathlon" class="llm-fu-bench-link" data-bench-id="toolathlon" title="View models and scores"><strong>Toolathlon</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>Agentic software tasks</td> <td class="desc">Long-horizon, real-world software tool-use tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-flash"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Flash</span> </a> </td> <td data-sort="49.4"> 49.4% </td> </tr> <tr data-bench="treebench" data-open-source="1"> <td style="text-align:center" data-sort="619">619</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=treebench" class="llm-fu-bench-link" data-bench-id="treebench" title="View models and scores"><strong>TreeBench</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://arxiv.org/abs/2507.07999" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Reasoning with tree structures</td> <td class="desc">Evaluates hierarchical/tree-structured reasoning and planning capabilities in LLMs/VLMs.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.6V</span> </a> </td> <td data-sort="51.4"> 51.4% </td> </tr> <tr data-bench="triqa" data-open-source="1"> <td style="text-align:center" data-sort="620">620</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=triqa" class="llm-fu-bench-link" data-bench-id="triqa" title="View models and scores"><strong>TriQA</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Knowledge QA</td> <td class="desc">Triadic question answering benchmark evaluating world knowledge and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Mixtral" data-model-version="Mixtral" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mixtral-8x22b"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mixtral 8x22B</span> </a> </td> <td data-sort="82.2"> 82.2% </td> </tr> <tr data-bench="triviaqa" data-open-source="1"> <td style="text-align:center" data-sort="621">621</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=triviaqa" class="llm-fu-bench-link" data-bench-id="triviaqa" title="View models and scores"><strong>TriviaQA</strong></a> <span class="score-symbol" title="52 scores" aria-label="52 scores">O</span> <a href="https://nlp.cs.washington.edu/triviaqa/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/1705.03551" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/mandarjoshi90/triviaqa" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Open-domain QA</td> <td class="desc">Open-domain question answering benchmark built from trivia and web evidence.</td> <td data-sort="1"> <span class="stars" title="Usage: 3/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemma" data-model-version="Gemma 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemma-3-pt-27b"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemma 3 PT 27B</span> </a> </td> <td data-sort="85.5"> 85.5% </td> </tr> <tr data-bench="triviaqa-wiki" data-open-source="1"> <td style="text-align:center" data-sort="622">622</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=triviaqa-wiki" class="llm-fu-bench-link" data-bench-id="triviaqa-wiki" title="View models and scores"><strong>TriviaQA-Wiki</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="http://nlp.cs.washington.edu/triviaqa/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Open-domain QA</td> <td class="desc">TriviaQA subset answering using Wikipedia evidence.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b-base"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B Base</span> </a> </td> <td data-sort="91.8"> 91.8% </td> </tr> <tr data-bench="trustllm" data-open-source="1"> <td style="text-align:center" data-sort="623">623</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=trustllm" class="llm-fu-bench-link" data-bench-id="trustllm" title="View models and scores"><strong>TrustLLM</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Safety / reliability</td> <td class="desc">TrustLLM benchmark for trustworthiness and safety behaviors.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-coder-480b-a35b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Coder-480B-A35B-Instruct</span> </a> </td> <td data-sort="88.4"> 88.4% </td> </tr> <tr data-bench="truthfulqa" data-open-source="1"> <td style="text-align:center" data-sort="624">624</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=truthfulqa" class="llm-fu-bench-link" data-bench-id="truthfulqa" title="View models and scores"><strong>TruthfulQA</strong></a> <span class="score-symbol" title="91 scores" aria-label="91 scores">O</span> <a href="https://github.com/sylinrl/TruthfulQA" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2109.07958" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Truthfulness / hallucination</td> <td class="desc">Measures whether a model imitates human falsehoods (truthfulness).</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Solar" data-model-version="Solar Mini" data-provider="upstage"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=solar-10.7b-instruct-v1.0"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">SOLAR-10.7B-Instruct-v1.0</span> </a> </td> <td data-sort="71.4"> 71.4% </td> </tr> <tr data-bench="truthfulqa-de" data-open-source="1"> <td style="text-align:center" data-sort="625">625</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=truthfulqa-de" class="llm-fu-bench-link" data-bench-id="truthfulqa-de" title="View models and scores"><strong>TruthfulQA (DE)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Truthfulness / hallucination (German)</td> <td class="desc">German translation of the TruthfulQA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b-instruct"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B Instruct</span> </a> </td> <td data-sort="0.2"> 0.2% </td> </tr> <tr data-bench="tvbench" data-open-source="0"> <td style="text-align:center" data-sort="626">626</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tvbench" class="llm-fu-bench-link" data-bench-id="tvbench" title="View models and scores"><strong>TVBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>TV comprehension</td> <td class="desc">Benchmark for TV show video comprehension and QA.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="71.5"> 71.5% </td> </tr> <tr data-bench="tydiqa" data-open-source="1"> <td style="text-align:center" data-sort="627">627</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=tydiqa" class="llm-fu-bench-link" data-bench-id="tydiqa" title="View models and scores"><strong>TydiQA</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> <a href="https://github.com/google-research-datasets/tydiqa" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Cross-lingual QA</td> <td class="desc">Typologically diverse QA across languages.</td> <td data-sort="2"> <span class="stars" title="Usage: 0/39; GitHub>100: yes; Recent: no; Leader: yes"> ★★<span class="empty">★★★</span> </span> </td> <td data-sort="313">313</td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.1" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.1-405b-base"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.1 405B Base</span> </a> </td> <td data-sort="34.3"> 34.3% </td> </tr> <tr data-bench="u-artifacts" data-open-source="0"> <td style="text-align:center" data-sort="628">628</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=u-artifacts" class="llm-fu-bench-link" data-bench-id="u-artifacts" title="View models and scores"><strong>U-Artifacts</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Agentic coding artifacts</td> <td class="desc">Benchmark focusing on generated code artifacts quality.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="57.8"> 57.8% </td> </tr> <tr data-bench="v-star" data-open-source="1"> <td style="text-align:center" data-sort="629">629</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=v-star" class="llm-fu-bench-link" data-bench-id="v-star" title="View models and scores"><strong>V*</strong></a> <span class="score-symbol" title="28 scores" aria-label="28 scores">O</span> </td> <td>Multimodal reasoning</td> <td class="desc">V* benchmark accuracy.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="95.8"> 95.8% </td> </tr> <tr data-bench="vcrbench" data-open-source="0"> <td style="text-align:center" data-sort="630">630</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vcrbench" class="llm-fu-bench-link" data-bench-id="vcrbench" title="View models and scores"><strong>VCRBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Visual commonsense reasoning</td> <td class="desc">Visual commonsense reasoning benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="59.8"> 59.8% </td> </tr> <tr data-bench="vct" data-open-source="0"> <td style="text-align:center" data-sort="631">631</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vct" class="llm-fu-bench-link" data-bench-id="vct" title="View models and scores"><strong>VCT</strong></a> <span class="score-symbol" title="26 scores" aria-label="26 scores">O</span> <a href="https://www.virologytest.ai/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://arxiv.org/abs/2504.16137" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Virology capability (protocol troubleshooting)</td> <td class="desc">Virology Capabilities Test: a benchmark that measures an LLM&#39;s ability to troubleshoot complex virology laboratory protocols.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="100.0"> 100.0% </td> </tr> <tr data-bench="vending-bench-2" data-open-source="0"> <td style="text-align:center" data-sort="632">632</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vending-bench-2" class="llm-fu-bench-link" data-bench-id="vending-bench-2" title="View models and scores"><strong>Vending-Bench 2</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Long-horizon agentic tasks</td> <td class="desc">Long-horizon agentic task benchmark evaluating sustained goal completion.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="5478.2"> 5478.2% </td> </tr> <tr data-bench="vibe-android" data-open-source="0"> <td style="text-align:center" data-sort="633">633</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vibe-android" class="llm-fu-bench-link" data-bench-id="vibe-android" title="View models and scores"><strong>Vibe Android</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Vibe evaluation (Android)</td> <td class="desc">Vibe evaluation on Android tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="92.2"> 92.2% </td> </tr> <tr data-bench="vibe-average" data-open-source="1"> <td style="text-align:center" data-sort="634">634</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vibe-average" class="llm-fu-bench-link" data-bench-id="vibe-average" title="View models and scores"><strong>Vibe Average</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> </td> <td>Vibe evaluation</td> <td class="desc">Aggregate Vibe evaluation score.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiniMax" data-model-version="MiniMax M2.1" data-provider="minimax"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minimax-m2.1"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiniMax M2.1</span> </a> </td> <td data-sort="88.6"> 88.6% </td> </tr> <tr data-bench="vibe-backend" data-open-source="0"> <td style="text-align:center" data-sort="635">635</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vibe-backend" class="llm-fu-bench-link" data-bench-id="vibe-backend" title="View models and scores"><strong>Vibe Backend</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Vibe evaluation (backend)</td> <td class="desc">Vibe evaluation on backend tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="98.0"> 98.0% </td> </tr> <tr data-bench="vibe-ios" data-open-source="0"> <td style="text-align:center" data-sort="636">636</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vibe-ios" class="llm-fu-bench-link" data-bench-id="vibe-ios" title="View models and scores"><strong>Vibe iOS</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Vibe evaluation (iOS)</td> <td class="desc">Vibe evaluation on iOS tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="90.0"> 90.0% </td> </tr> <tr data-bench="vibe-simulation" data-open-source="0"> <td style="text-align:center" data-sort="637">637</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vibe-simulation" class="llm-fu-bench-link" data-bench-id="vibe-simulation" title="View models and scores"><strong>Vibe Simulation</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Vibe evaluation (simulation)</td> <td class="desc">Vibe evaluation on simulation tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="89.2"> 89.2% </td> </tr> <tr data-bench="vibe-web" data-open-source="1"> <td style="text-align:center" data-sort="638">638</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vibe-web" class="llm-fu-bench-link" data-bench-id="vibe-web" title="View models and scores"><strong>Vibe Web</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Vibe evaluation (web)</td> <td class="desc">Vibe evaluation on web tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiniMax" data-model-version="MiniMax M2.1" data-provider="minimax"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=minimax-m2.1"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiniMax M2.1</span> </a> </td> <td data-sort="91.5"> 91.5% </td> </tr> <tr data-bench="vibeval" data-open-source="0"> <td style="text-align:center" data-sort="639">639</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vibeval" class="llm-fu-bench-link" data-bench-id="vibeval" title="View models and scores"><strong>VibeEval</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Aesthetic/visual quality</td> <td class="desc">VLM aesthetic evaluation with GPT scores.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="76.4"> 76.4% </td> </tr> <tr data-bench="video-mme" data-open-source="1"> <td style="text-align:center" data-sort="640">640</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=video-mme" class="llm-fu-bench-link" data-bench-id="video-mme" title="View models and scores"><strong>Video-MME</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> <a href="https://video-mme.github.io/home_page.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Video understanding (multimodal)</td> <td class="desc">Multimodal evaluation of video understanding and reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-32b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL 32B Instruct</span> </a> </td> <td data-sort="76.6"> 76.6% </td> </tr> <tr data-bench="videoholmes" data-open-source="0"> <td style="text-align:center" data-sort="641">641</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=videoholmes" class="llm-fu-bench-link" data-bench-id="videoholmes" title="View models and scores"><strong>VideoHolmes</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Video QA</td> <td class="desc">Video question answering benchmark focused on detective-style clues.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="65.5"> 65.5% </td> </tr> <tr data-bench="videomme" data-open-source="0"> <td style="text-align:center" data-sort="642">642</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=videomme" class="llm-fu-bench-link" data-bench-id="videomme" title="View models and scores"><strong>VideoMME</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Multimodal video evaluation</td> <td class="desc">Video multimodal evaluation suite (VideoMME).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="88.4"> 88.4% </td> </tr> <tr data-bench="videomme-nosub" data-open-source="0"> <td style="text-align:center" data-sort="643">643</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=videomme-nosub" class="llm-fu-bench-link" data-bench-id="videomme-nosub" title="View models and scores"><strong>VideoMME (w/o sub)</strong></a> <span class="score-symbol" title="32 scores" aria-label="32 scores">O</span> </td> <td>Video understanding</td> <td class="desc">Video understanding benchmark without subtitles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="87.7"> 87.7% </td> </tr> <tr data-bench="videomme-sub" data-open-source="0"> <td style="text-align:center" data-sort="644">644</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=videomme-sub" class="llm-fu-bench-link" data-bench-id="videomme-sub" title="View models and scores"><strong>VideoMME (w/sub)</strong></a> <span class="score-symbol" title="15 scores" aria-label="15 scores">O</span> </td> <td>Video understanding</td> <td class="desc">Video understanding benchmark with subtitles.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="88.4"> 88.4% </td> </tr> <tr data-bench="videommmu" data-open-source="0"> <td style="text-align:center" data-sort="645">645</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=videommmu" class="llm-fu-bench-link" data-bench-id="videommmu" title="View models and scores"><strong>VideoMMMU</strong></a> <span class="score-symbol" title="50 scores" aria-label="50 scores">O</span> </td> <td>Multimodal video understanding</td> <td class="desc">Video-based extension of MMMU evaluating temporal multimodal reasoning and perception across disciplines.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="87.6"> 87.6% </td> </tr> <tr data-bench="videoreasonbench" data-open-source="0"> <td style="text-align:center" data-sort="646">646</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=videoreasonbench" class="llm-fu-bench-link" data-bench-id="videoreasonbench" title="View models and scores"><strong>VideoReasonBench</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Video reasoning</td> <td class="desc">Video reasoning benchmark assessing temporal and causal understanding.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="59.7"> 59.7% </td> </tr> <tr data-bench="videosimpleqa" data-open-source="0"> <td style="text-align:center" data-sort="647">647</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=videosimpleqa" class="llm-fu-bench-link" data-bench-id="videosimpleqa" title="View models and scores"><strong>VideoSimpleQA</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Video QA</td> <td class="desc">Simple question answering over short videos.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="71.9"> 71.9% </td> </tr> <tr data-bench="vispeak" data-open-source="0"> <td style="text-align:center" data-sort="648">648</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vispeak" class="llm-fu-bench-link" data-bench-id="vispeak" title="View models and scores"><strong>ViSpeak</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Video dialogue</td> <td class="desc">Video-grounded dialogue and description benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="89.0"> 89.0% </td> </tr> <tr data-bench="visualpuzzle" data-open-source="0"> <td style="text-align:center" data-sort="649">649</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=visualpuzzle" class="llm-fu-bench-link" data-bench-id="visualpuzzle" title="View models and scores"><strong>VisualPuzzle</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Visual reasoning</td> <td class="desc">Visual puzzle solving benchmark evaluating reasoning and pattern recognition capabilities.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="57.8"> 57.8% </td> </tr> <tr data-bench="visualwebbench-avg" data-open-source="1"> <td style="text-align:center" data-sort="650">650</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=visualwebbench-avg" class="llm-fu-bench-link" data-bench-id="visualwebbench-avg" title="View models and scores"><strong>VisualWebBench</strong></a> <span class="score-symbol" title="28 scores" aria-label="28 scores">O</span> <a href="https://arxiv.org/abs/2404.05955" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/VisualWebBench/VisualWebBench" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Web UI understanding</td> <td class="desc">Average accuracy on VisualWebBench.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Holo" data-model-version="Holo1.5" data-provider="hcompany"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=holo1.5-72b"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Holo1.5-72B</span> </a> </td> <td data-sort="83.8"> 83.8% </td> </tr> <tr data-bench="visulogic" data-open-source="1"> <td style="text-align:center" data-sort="651">651</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=visulogic" class="llm-fu-bench-link" data-bench-id="visulogic" title="View models and scores"><strong>VisuLogic</strong></a> <span class="score-symbol" title="19 scores" aria-label="19 scores">O</span> <a href="https://visulogic-benchmark.github.io/VisuLogic/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://visulogic-benchmark.github.io/VisuLogic/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2504.15279" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/VisuLogic-Benchmark/VisuLogic-Eval" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Visual logical reasoning</td> <td class="desc">Logical reasoning and compositionality benchmark for visual-language models.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="ERNIE" data-model-version="ERNIE 4.5 VL" data-provider="baidu"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=ernie-4.5-vl-28b-a3b-thinking"> <span class="brand"> <img class="logo" alt="baidu logo" data-company="baidu" data-country="China" src="https://furukama.com/logos/baidu.svg?v=c388ef3a" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">ERNIE-4.5-VL-28B-A3B-Thinking</span> </a> </td> <td data-sort="52.5"> 52.5% </td> </tr> <tr data-bench="vitabench" data-open-source="0"> <td style="text-align:center" data-sort="652">652</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vitabench" class="llm-fu-bench-link" data-bench-id="vitabench" title="View models and scores"><strong>VitaBench</strong></a> <span class="score-symbol" title="29 scores" aria-label="29 scores">O</span> </td> <td>Industry QA</td> <td class="desc">Industry-focused benchmark evaluating domain QA performance.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-opus-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Opus 4.5</span> </a> </td> <td data-sort="56.3"> 56.3% </td> </tr> <tr data-bench="vl-rewardbench" data-open-source="0"> <td style="text-align:center" data-sort="653">653</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vl-rewardbench" class="llm-fu-bench-link" data-bench-id="vl-rewardbench" title="View models and scores"><strong>VL-RewardBench</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Reward modeling (VL)</td> <td class="desc">Reward alignment benchmark for VLMs.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 3.7" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-3.7-sonnet"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude 3.7 Sonnet</span> </a> </td> <td data-sort="67.4"> 67.4% </td> </tr> <tr data-bench="vlms-are-biased" data-open-source="0"> <td style="text-align:center" data-sort="654">654</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vlms-are-biased" class="llm-fu-bench-link" data-bench-id="vlms-are-biased" title="View models and scores"><strong>VLMs are Biased</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> <a href="https://vlmsarebiased.github.io/" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> <a href="https://vlmsarebiased.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2505.23941" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/anvo25/vlms-are-biased" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Multimodal bias</td> <td class="desc">Evaluates whether VLMs truly &#39;see&#39; vs. relying on memorized knowledge; measures bias toward non-visual priors.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="90">90</td> <td class="leader-cell" data-model-family="o4" data-model-version="o4" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=o4-mini"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">o4 mini</span> </a> </td> <td data-sort="20.2"> 20.2% </td> </tr> <tr data-bench="vlms-are-blind" data-open-source="1"> <td style="text-align:center" data-sort="655">655</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vlms-are-blind" class="llm-fu-bench-link" data-bench-id="vlms-are-blind" title="View models and scores"><strong>VLMs are Blind</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> <a href="https://vlmsareblind.github.io/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://github.com/anguyen8/vision-llms-are-blind" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Visual grounding robustness</td> <td class="desc">Evaluates failure modes of VLMs in grounding and perception tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="MiMo" data-model-version="MiMo VL" data-provider="xiaomi"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mimo-vl-7b-rl"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">MiMo-VL 7B-RL</span> </a> </td> <td data-sort="79.4"> 79.4% </td> </tr> <tr data-bench="vlmsarebiased" data-open-source="0"> <td style="text-align:center" data-sort="656">656</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vlmsarebiased" class="llm-fu-bench-link" data-bench-id="vlmsarebiased" title="View models and scores"><strong>VLMsAreBiased</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Multimodal bias</td> <td class="desc">Benchmark evaluating biases in vision-language models.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Seed" data-model-version="Seed 1.8" data-provider="bytedance"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=seed1.8"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Seed1.8</span> </a> </td> <td data-sort="62.0"> 62.0% </td> </tr> <tr data-bench="vlmsareblind" data-open-source="0"> <td style="text-align:center" data-sort="657">657</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vlmsareblind" class="llm-fu-bench-link" data-bench-id="vlmsareblind" title="View models and scores"><strong>VLMsAreBlind</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>Multimodal robustness</td> <td class="desc">Benchmark probing robustness of vision-language models to visual perturbations.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="97.5"> 97.5% </td> </tr> <tr data-bench="voicebench-advbench" data-open-source="1"> <td style="text-align:center" data-sort="658">658</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-advbench" class="llm-fu-bench-link" data-bench-id="voicebench-advbench" title="View models and scores"><strong>VoiceBench AdvBench</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench adversarial safety evaluation.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-omni-30b-a3b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Omni-30B-A3B-Thinking</span> </a> </td> <td data-sort="99.4"> 99.4% </td> </tr> <tr data-bench="voicebench-alpacaeval" data-open-source="1"> <td style="text-align:center" data-sort="659">659</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-alpacaeval" class="llm-fu-bench-link" data-bench-id="voicebench-alpacaeval" title="View models and scores"><strong>VoiceBench AlpacaEval</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench evaluation on AlpacaEval instructions.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-omni-flash-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Omni-Flash-Thinking</span> </a> </td> <td data-sort="96.8"> 96.8% </td> </tr> <tr data-bench="voicebench-bbh" data-open-source="0"> <td style="text-align:center" data-sort="660">660</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-bbh" class="llm-fu-bench-link" data-bench-id="voicebench-bbh" title="View models and scores"><strong>VoiceBench BBH</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench evaluation on Big-Bench Hard prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="92.6"> 92.6% </td> </tr> <tr data-bench="voicebench-commoneval" data-open-source="1"> <td style="text-align:center" data-sort="661">661</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-commoneval" class="llm-fu-bench-link" data-bench-id="voicebench-commoneval" title="View models and scores"><strong>VoiceBench CommonEval</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench evaluation on CommonEval.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-omni-flash-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Omni-Flash-Instruct</span> </a> </td> <td data-sort="91.0"> 91.0% </td> </tr> <tr data-bench="voicebench-ifeval" data-open-source="0"> <td style="text-align:center" data-sort="662">662</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-ifeval" class="llm-fu-bench-link" data-bench-id="voicebench-ifeval" title="View models and scores"><strong>VoiceBench IFEval</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench instruction-following evaluation (IFEval).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="85.7"> 85.7% </td> </tr> <tr data-bench="voicebench-mau-v05-15-25" data-open-source="1"> <td style="text-align:center" data-sort="663">663</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-mau-v05-15-25" class="llm-fu-bench-link" data-bench-id="voicebench-mau-v05-15-25" title="View models and scores"><strong>MMAU v05.15.25</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Audio reasoning</td> <td class="desc">Audio reasoning benchmark MMAU v05.15.25.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-omni-flash-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Omni-Flash-Instruct</span> </a> </td> <td data-sort="77.6"> 77.6% </td> </tr> <tr data-bench="voicebench-mmsu" data-open-source="1"> <td style="text-align:center" data-sort="664">664</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-mmsu" class="llm-fu-bench-link" data-bench-id="voicebench-mmsu" title="View models and scores"><strong>VoiceBench MMSU</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench MMSU benchmark (voice modality).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-omni-flash-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Omni-Flash-Thinking</span> </a> </td> <td data-sort="84.3"> 84.3% </td> </tr> <tr data-bench="voicebench-mmsu-audio" data-open-source="0"> <td style="text-align:center" data-sort="665">665</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-mmsu-audio" class="llm-fu-bench-link" data-bench-id="voicebench-mmsu-audio" title="View models and scores"><strong>VoiceBench MMSU (Audio)</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>Audio reasoning</td> <td class="desc">Audio reasoning MMSU results.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="77.7"> 77.7% </td> </tr> <tr data-bench="voicebench-openbookqa" data-open-source="1"> <td style="text-align:center" data-sort="666">666</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-openbookqa" class="llm-fu-bench-link" data-bench-id="voicebench-openbookqa" title="View models and scores"><strong>VoiceBench OpenBookQA</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench results on OpenBookQA prompts.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-omni-flash-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-Omni-Flash-Thinking</span> </a> </td> <td data-sort="95.0"> 95.0% </td> </tr> <tr data-bench="voicebench-sd-qa" data-open-source="0"> <td style="text-align:center" data-sort="667">667</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-sd-qa" class="llm-fu-bench-link" data-bench-id="voicebench-sd-qa" title="View models and scores"><strong>VoiceBench SD-QA</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench Spoken Dialogue QA results.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="90.1"> 90.1% </td> </tr> <tr data-bench="voicebench-wildvoice" data-open-source="0"> <td style="text-align:center" data-sort="668">668</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=voicebench-wildvoice" class="llm-fu-bench-link" data-bench-id="voicebench-wildvoice" title="View models and scores"><strong>VoiceBench WildVoice</strong></a> <span class="score-symbol" title="13 scores" aria-label="13 scores">O</span> </td> <td>VoiceBench</td> <td class="desc">VoiceBench evaluation on WildVoice dataset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 2.5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-2.5-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 2.5 Pro</span> </a> </td> <td data-sort="93.4"> 93.4% </td> </tr> <tr data-bench="vpct" data-open-source="0"> <td style="text-align:center" data-sort="669">669</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vpct" class="llm-fu-bench-link" data-bench-id="vpct" title="View models and scores"><strong>VPCT</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Multimodal reasoning</td> <td class="desc">Visual perception and comprehension test.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="90.0"> 90.0% </td> </tr> <tr data-bench="vqav2" data-open-source="1"> <td style="text-align:center" data-sort="670">670</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vqav2" class="llm-fu-bench-link" data-bench-id="vqav2" title="View models and scores"><strong>VQAv2</strong></a> <span class="score-symbol" title="53 scores" aria-label="53 scores">O</span> <a href="https://visualqa.org/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Visual question answering</td> <td class="desc">Standard Visual Question Answering v2 benchmark on natural images.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Molmo2" data-model-version="Molmo2" data-provider="allenai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=molmo2-8b"> <span class="brand"> <img class="logo" alt="allenai logo" data-company="allenai" data-country="United States" src="https://furukama.com/logos/allenai.svg?v=27b5cbbf" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Molmo2-8B</span> </a> </td> <td data-sort="87.0"> 87.0% </td> </tr> <tr data-bench="vsi-bench" data-open-source="1"> <td style="text-align:center" data-sort="671">671</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=vsi-bench" class="llm-fu-bench-link" data-bench-id="vsi-bench" title="View models and scores"><strong>VSI-Bench</strong></a> <span class="score-symbol" title="11 scores" aria-label="11 scores">O</span> </td> <td>Spatial intelligence</td> <td class="desc">Visual spatial intelligence benchmark covering 3D reasoning and spatial inference tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-30b-a3b-instruct"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL-30B-A3B Instruct</span> </a> </td> <td data-sort="63.2"> 63.2% </td> </tr> <tr data-bench="webclick" data-open-source="0"> <td style="text-align:center" data-sort="672">672</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=webclick" class="llm-fu-bench-link" data-bench-id="webclick" title="View models and scores"><strong>WebClick</strong></a> <span class="score-symbol" title="12 scores" aria-label="12 scores">O</span> </td> <td>GUI agents</td> <td class="desc">Task success on the WebClick UI agent benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4</span> </a> </td> <td data-sort="93.0"> 93.0% </td> </tr> <tr data-bench="webdev-arena" data-open-source="0"> <td style="text-align:center" data-sort="673">673</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=webdev-arena" class="llm-fu-bench-link" data-bench-id="webdev-arena" title="View models and scores"><strong>WebDev Arena</strong></a> <span class="score-symbol" title="57 scores" aria-label="57 scores">O</span> <a href="https://lmarena.ai/leaderboard/webdev" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Web development agents</td> <td class="desc">Arena evaluation for autonomous web development agents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5</span> </a> </td> <td data-sort="1483"> 1483 </td> </tr> <tr data-bench="webquest-multiqa" data-open-source="1"> <td style="text-align:center" data-sort="674">674</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=webquest-multiqa" class="llm-fu-bench-link" data-bench-id="webquest-multiqa" title="View models and scores"><strong>WebQuest-MultiQA</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Web agents</td> <td class="desc">Multi-question web search and interaction tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.5V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.5v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.5V</span> </a> </td> <td data-sort="60.6"> 60.6% </td> </tr> <tr data-bench="webquest-singleqa" data-open-source="1"> <td style="text-align:center" data-sort="675">675</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=webquest-singleqa" class="llm-fu-bench-link" data-bench-id="webquest-singleqa" title="View models and scores"><strong>WebQuest-SingleQA</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Web agents</td> <td class="desc">Single-question web search and interaction tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.6V</span> </a> </td> <td data-sort="79.5"> 79.5% </td> </tr> <tr data-bench="websrc" data-open-source="1"> <td style="text-align:center" data-sort="676">676</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=websrc" class="llm-fu-bench-link" data-bench-id="websrc" title="View models and scores"><strong>WebSrc</strong></a> <span class="score-symbol" title="18 scores" aria-label="18 scores">O</span> </td> <td>Web QA</td> <td class="desc">Webpage question answering (SQuAD F1).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Holo" data-model-version="Holo1.5" data-provider="hcompany"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=holo1.5-72b"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Holo1.5-72B</span> </a> </td> <td data-sort="97.2"> 97.2% </td> </tr> <tr data-bench="webvoyager" data-open-source="1"> <td style="text-align:center" data-sort="677">677</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=webvoyager" class="llm-fu-bench-link" data-bench-id="webvoyager" title="View models and scores"><strong>WebVoyager</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Web agents</td> <td class="desc">Web navigation and interaction tasks for LLM agents.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.6V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.6v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.6V</span> </a> </td> <td data-sort="81.0"> 81.0% </td> </tr> <tr data-bench="webvoyager2" data-open-source="1"> <td style="text-align:center" data-sort="678">678</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=webvoyager2" class="llm-fu-bench-link" data-bench-id="webvoyager2" title="View models and scores"><strong>WebVoyager2</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> </td> <td>Web agents</td> <td class="desc">Web navigation and interaction tasks for LLM agents (v2).</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.5V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.5v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.5V</span> </a> </td> <td data-sort="84.4"> 84.4% </td> </tr> <tr data-bench="webwalker-qa" data-open-source="0"> <td style="text-align:center" data-sort="679">679</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=webwalker-qa" class="llm-fu-bench-link" data-bench-id="webwalker-qa" title="View models and scores"><strong>WebWalkerQA</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Web agents</td> <td class="desc">WebWalker tasks evaluating autonomous browsing question answering performance.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Tongyi" data-model-version="Tongyi" data-provider="tongyi"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=tongyi-deepresearch"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Tongyi DeepResearch</span> </a> </td> <td data-sort="72.2"> 72.2% </td> </tr> <tr data-bench="wemath" data-open-source="1"> <td style="text-align:center" data-sort="680">680</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wemath" class="llm-fu-bench-link" data-bench-id="wemath" title="View models and scores"><strong>WeMath</strong></a> <span class="score-symbol" title="17 scores" aria-label="17 scores">O</span> </td> <td>Math reasoning</td> <td class="desc">Math reasoning benchmark spanning diverse curricula and difficulty levels.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="87.9"> 87.9% </td> </tr> <tr data-bench="wide-search" data-open-source="0"> <td style="text-align:center" data-sort="681">681</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wide-search" class="llm-fu-bench-link" data-bench-id="wide-search" title="View models and scores"><strong>WideSearch</strong></a> <span class="score-symbol" title="19 scores" aria-label="19 scores">O</span> </td> <td>Web search</td> <td class="desc">Wide web search and QA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5.2" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5.2"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5.2</span> </a> </td> <td data-sort="76.8"> 76.8% </td> </tr> <tr data-bench="wild-jailbreak" data-open-source="1"> <td style="text-align:center" data-sort="682">682</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wild-jailbreak" class="llm-fu-bench-link" data-bench-id="wild-jailbreak" title="View models and scores"><strong>Wild-Jailbreak</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Safety / jailbreak</td> <td class="desc">Adversarial jailbreak benchmark evaluating refusal robustness.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-OSS" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-oss-120b-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-OSS 120B (High)</span> </a> </td> <td data-sort="98.2"> 98.2% </td> </tr> <tr data-bench="wildbench-v2" data-open-source="1"> <td style="text-align:center" data-sort="683">683</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wildbench-v2" class="llm-fu-bench-link" data-bench-id="wildbench-v2" title="View models and scores"><strong>WildBench V2</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Instruction following</td> <td class="desc">WildBench V2 human preference benchmark for instruction following and helpfulness.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Mistral" data-model-version="Mistral 3.2" data-provider="mistral"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=mistral-small-3.2-24b-instruct"> <span class="brand"> <img class="logo" alt="mistral logo" data-company="mistral" data-country="France" src="https://furukama.com/logos/mistral.svg?v=9aa5d8f2" /> <span class="flag" role="img" aria-label="France" title="France" data-country="France">🇫🇷</span> </span> <span class="leader-name">Mistral Small 3.2 24B Instruct</span> </a> </td> <td data-sort="65.3"> 65.3% </td> </tr> <tr data-bench="wildguardtest" data-open-source="1"> <td style="text-align:center" data-sort="684">684</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wildguardtest" class="llm-fu-bench-link" data-bench-id="wildguardtest" title="View models and scores"><strong>WildGuardTest</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Safety</td> <td class="desc">WildGuardTest safety benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Thinking</span> </a> </td> <td data-sort="86.8"> 86.8% </td> </tr> <tr data-bench="winogender" data-open-source="1"> <td style="text-align:center" data-sort="685">685</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=winogender" class="llm-fu-bench-link" data-bench-id="winogender" title="View models and scores"><strong>Winogender</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> <a href="https://github.com/rudinger/winogender-schemas" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Gender bias (coreference)</td> <td class="desc">Coreference resolution dataset for measuring gender bias.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b-instruct"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B Instruct</span> </a> </td> <td data-sort="84.3"> 84.3% </td> </tr> <tr data-bench="winogrande" data-open-source="1"> <td style="text-align:center" data-sort="686">686</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=winogrande" class="llm-fu-bench-link" data-bench-id="winogrande" title="View models and scores"><strong>WinoGrande</strong></a> <span class="score-symbol" title="118 scores" aria-label="118 scores">O</span> <a href="https://github.com/allenai/winogrande" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Coreference reasoning</td> <td class="desc">Large-scale adversarial Winograd Schema-style pronoun resolution.</td> <td data-sort="1"> <span class="stars" title="Usage: 4/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="99">99</td> <td class="leader-cell" data-model-family="OLMo" data-model-version="OLMo 3" data-provider="allenai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=olmo-3-think-32b"> <span class="brand"> <img class="logo" alt="allenai logo" data-company="allenai" data-country="United States" src="https://furukama.com/logos/allenai.svg?v=27b5cbbf" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">OLMo-3-Think-32B</span> </a> </td> <td data-sort="90.3"> 90.3% </td> </tr> <tr data-bench="winogrande-de" data-open-source="1"> <td style="text-align:center" data-sort="687">687</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=winogrande-de" class="llm-fu-bench-link" data-bench-id="winogrande-de" title="View models and scores"><strong>WinoGrande (DE)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Coreference reasoning (German)</td> <td class="desc">German translation of the WinoGrande pronoun resolution benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="0.8"> 0.8% </td> </tr> <tr data-bench="wmdp-bio" data-open-source="1"> <td style="text-align:center" data-sort="688">688</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wmdp-bio" class="llm-fu-bench-link" data-bench-id="wmdp-bio" title="View models and scores"><strong>WMDP Bio</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://www.wmdp.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2403.03218" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/wmdp" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Biosecurity knowledge</td> <td class="desc">Weapons of Mass Destruction Proxy benchmark for biosecurity, measuring hazardous biological knowledge without info hazards.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Zephyr" data-model-version="Zephyr" data-provider="huggingface"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=zephyr-7b"> <span class="brand"> <img class="logo no-invert" alt="huggingface logo" data-company="huggingface" data-country="United States" src="https://furukama.com/logos/huggingface.svg?v=5cc8e86e" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Zephyr 7B</span> </a> </td> <td data-sort="63.7"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 63.7%</span> </td> </tr> <tr data-bench="wmdp-chem" data-open-source="1"> <td style="text-align:center" data-sort="689">689</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wmdp-chem" class="llm-fu-bench-link" data-bench-id="wmdp-chem" title="View models and scores"><strong>WMDP Chem</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://www.wmdp.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2403.03218" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/wmdp" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Chemical security knowledge</td> <td class="desc">WMDP benchmark for chemical security, evaluating knowledge relevant to chemical weapons development.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Zephyr" data-model-version="Zephyr" data-provider="huggingface"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=zephyr-7b"> <span class="brand"> <img class="logo no-invert" alt="huggingface logo" data-company="huggingface" data-country="United States" src="https://furukama.com/logos/huggingface.svg?v=5cc8e86e" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Zephyr 7B</span> </a> </td> <td data-sort="45.8"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 45.8%</span> </td> </tr> <tr data-bench="wmdp-cyber" data-open-source="1"> <td style="text-align:center" data-sort="690">690</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wmdp-cyber" class="llm-fu-bench-link" data-bench-id="wmdp-cyber" title="View models and scores"><strong>WMDP Cyber</strong></a> <span class="score-symbol" title="3 scores" aria-label="3 scores">o</span> <a href="https://www.wmdp.ai/" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/2403.03218" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> <a href="https://github.com/centerforaisafety/wmdp" target="_blank" rel="noopener" class="gh-icon" title="GitHub" aria-label="GitHub"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M8 0C3.58 0 0 3.58 0 8a8 8 0 0 0 5.47 7.59c.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2 .37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.58.82-2.14-.08-.2-.36-1.01.08-2.11 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.91.08 2.11.51.56.82 1.27.82 2.14 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.45.55.38A8 8 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path> </svg> </a> </td> <td>Cybersecurity knowledge</td> <td class="desc">WMDP benchmark for cybersecurity, assessing knowledge that could aid in cyber weapons development.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Zephyr" data-model-version="Zephyr" data-provider="huggingface"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=zephyr-7b"> <span class="brand"> <img class="logo no-invert" alt="huggingface logo" data-company="huggingface" data-country="United States" src="https://furukama.com/logos/huggingface.svg?v=5cc8e86e" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Zephyr 7B</span> </a> </td> <td data-sort="44.0"> <span title="Lower is better" aria-label="Lower is better" style="white-space:nowrap">↓ 44.0%</span> </td> </tr> <tr data-bench="wmt16" data-open-source="1"> <td style="text-align:center" data-sort="691">691</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wmt16" class="llm-fu-bench-link" data-bench-id="wmt16" title="View models and scores"><strong>WMT16 En–De</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> <a href="https://www.statmt.org/wmt16/translation-task.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Machine translation</td> <td class="desc">WMT16 English–German translation benchmark (news).</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b-instruct"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B Instruct</span> </a> </td> <td data-sort="38.8"> 38.8% </td> </tr> <tr data-bench="wmt16-instruct" data-open-source="1"> <td style="text-align:center" data-sort="692">692</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wmt16-instruct" class="llm-fu-bench-link" data-bench-id="wmt16-instruct" title="View models and scores"><strong>WMT16 En–De (Instruct)</strong></a> <span class="score-symbol" title="2 scores" aria-label="2 scores">o</span> </td> <td>Machine translation</td> <td class="desc">Instruction-tuned evaluation on the WMT16 English–German translation set.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b-instruct"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B Instruct</span> </a> </td> <td data-sort="37.9"> 37.9% </td> </tr> <tr data-bench="wmt24-plus-plus" data-open-source="1"> <td style="text-align:center" data-sort="693">693</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wmt24-plus-plus" class="llm-fu-bench-link" data-bench-id="wmt24-plus-plus" title="View models and scores"><strong>WMT24&#43;&#43;</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> <a href="https://arxiv.org/abs/2502.12404v1" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Machine translation</td> <td class="desc">Extended WMT 2024 evaluation across multiple language pairs.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-235b-a22b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-235B-A22B-Thinking-2507</span> </a> </td> <td data-sort="94.7"> 94.7% </td> </tr> <tr data-bench="worldtravel2-multimodal" data-open-source="0"> <td style="text-align:center" data-sort="694">694</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=worldtravel2-multimodal" class="llm-fu-bench-link" data-bench-id="worldtravel2-multimodal" title="View models and scores"><strong>WorldTravel2 (multi-modal)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Travel planning (multimodal)</td> <td class="desc">WorldTravel2 benchmark multimodal track.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="47.2"> 47.2% </td> </tr> <tr data-bench="worldtravel2-text" data-open-source="0"> <td style="text-align:center" data-sort="695">695</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=worldtravel2-text" class="llm-fu-bench-link" data-bench-id="worldtravel2-text" title="View models and scores"><strong>WorldTravel2 (text)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Travel planning (text)</td> <td class="desc">WorldTravel2 benchmark text-only track.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="56.4"> 56.4% </td> </tr> <tr data-bench="worldvqa" data-open-source="0"> <td style="text-align:center" data-sort="696">696</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=worldvqa" class="llm-fu-bench-link" data-bench-id="worldvqa" title="View models and scores"><strong>WorldVQA</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>World knowledge VQA</td> <td class="desc">Visual question answering requiring world knowledge and commonsense reasoning.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemini" data-model-version="Gemini 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemini-3-pro"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemini 3 Pro</span> </a> </td> <td data-sort="47.4"> 47.4% </td> </tr> <tr data-bench="writingbench" data-open-source="1"> <td style="text-align:center" data-sort="697">697</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=writingbench" class="llm-fu-bench-link" data-bench-id="writingbench" title="View models and scores"><strong>WritingBench</strong></a> <span class="score-symbol" title="27 scores" aria-label="27 scores">O</span> </td> <td>Writing quality</td> <td class="desc">General-purpose writing quality benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-235b-a22b-thinking-2507"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-235B-A22B-Thinking-2507</span> </a> </td> <td data-sort="88.3"> 88.3% </td> </tr> <tr data-bench="wsc" data-open-source="1"> <td style="text-align:center" data-sort="698">698</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=wsc" class="llm-fu-bench-link" data-bench-id="wsc" title="View models and scores"><strong>WSC</strong></a> <span class="score-symbol" title="14 scores" aria-label="14 scores">O</span> <a href="https://cs.nyu.edu/faculty/davise/papers/WinogradSchemas/WS.html" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> <a href="https://arxiv.org/abs/1109.1334" target="_blank" rel="noopener" class="doc-icon" title="PDF" aria-label="PDF"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M4 1h6l4 4v9a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1zm6 1.5V5h3.5L10 2.5z"></path> <rect x="5" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="5" y="8.6" width="1.2" height="0.9" rx="0.2"></rect> <rect x="7.2" y="7.2" width="1.6" height="0.9" rx="0.2"></rect> <rect x="7.2" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> <rect x="9.1" y="7.2" width="1.9" height="0.9" rx="0.2"></rect> <rect x="9.1" y="8.6" width="0.9" height="2.0" rx="0.2"></rect> </svg> </a> </td> <td>Coreference reasoning</td> <td class="desc">Classic Winograd Schema Challenge measuring commonsense coreference.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Gemma" data-model-version="Gemma 3" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gemma-3-pt-27b"> <span class="brand"> <img class="logo no-invert" alt="google logo" data-company="google" data-country="United States" src="https://furukama.com/logos/google.svg?v=e073e8cd" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Gemma 3 PT 27B</span> </a> </td> <td data-sort="91.9"> 91.9% </td> </tr> <tr data-bench="xbench-deepsearch" data-open-source="0"> <td style="text-align:center" data-sort="699">699</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xbench-deepsearch" class="llm-fu-bench-link" data-bench-id="xbench-deepsearch" title="View models and scores"><strong>xBench-DeepSearch</strong></a> <span class="score-symbol" title="23 scores" aria-label="23 scores">O</span> </td> <td>Agentic research</td> <td class="desc">Evaluates multi-hop deep research workflows on xBench DeepSearch tasks.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="77.9"> 77.9% </td> </tr> <tr data-bench="xbench-deepsearch-2025.05" data-open-source="1"> <td style="text-align:center" data-sort="700">700</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xbench-deepsearch-2025.05" class="llm-fu-bench-link" data-bench-id="xbench-deepsearch-2025.05" title="View models and scores"><strong>xBench-DeepSearch (2025.05)</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> </td> <td>Agentic research</td> <td class="desc">xBench DeepSearch benchmark May 2025 snapshot.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step 3.5 Flash" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step-3.5-flash-20260204"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step-3.5 Flash 20260204</span> </a> </td> <td data-sort="83.7"> 83.7% </td> </tr> <tr data-bench="xbench-deepsearch-2025.10" data-open-source="1"> <td style="text-align:center" data-sort="701">701</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xbench-deepsearch-2025.10" class="llm-fu-bench-link" data-bench-id="xbench-deepsearch-2025.10" title="View models and scores"><strong>xBench-DeepSearch (2025.10)</strong></a> <span class="score-symbol" title="6 scores" aria-label="6 scores">o</span> </td> <td>Agentic research</td> <td class="desc">xBench DeepSearch benchmark October 2025 snapshot.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Step" data-model-version="Step 3.5 Flash" data-provider="stepfun"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=step-3.5-flash-20260204"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Step-3.5 Flash 20260204</span> </a> </td> <td data-sort="56.3"> 56.3% </td> </tr> <tr data-bench="xlrs-bench" data-open-source="1"> <td style="text-align:center" data-sort="702">702</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xlrs-bench" class="llm-fu-bench-link" data-bench-id="xlrs-bench" title="View models and scores"><strong>XLRS-Bench</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Remote sensing</td> <td class="desc">Remote sensing benchmark for evaluating multimodal AI on satellite and aerial imagery.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Intern" data-model-version="Intern-S1" data-provider="internlm"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=intern-s1-pro"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">Intern-S1-Pro</span> </a> </td> <td data-sort="52.8"> 52.8% </td> </tr> <tr data-bench="xpertbench-edu" data-open-source="0"> <td style="text-align:center" data-sort="703">703</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xpertbench-edu" class="llm-fu-bench-link" data-bench-id="xpertbench-edu" title="View models and scores"><strong>XpertBench (Edu)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Economics/education</td> <td class="desc">XpertBench education domain subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="56.9"> 56.9% </td> </tr> <tr data-bench="xpertbench-fin" data-open-source="0"> <td style="text-align:center" data-sort="704">704</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xpertbench-fin" class="llm-fu-bench-link" data-bench-id="xpertbench-fin" title="View models and scores"><strong>XpertBench (Fin)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Economics/finance</td> <td class="desc">XpertBench finance domain subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="64.5"> 64.5% </td> </tr> <tr data-bench="xpertbench-humanities" data-open-source="0"> <td style="text-align:center" data-sort="705">705</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xpertbench-humanities" class="llm-fu-bench-link" data-bench-id="xpertbench-humanities" title="View models and scores"><strong>XpertBench (Humanities)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Economics/humanities</td> <td class="desc">XpertBench humanities domain subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="68.5"> 68.5% </td> </tr> <tr data-bench="xpertbench-law" data-open-source="0"> <td style="text-align:center" data-sort="706">706</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xpertbench-law" class="llm-fu-bench-link" data-bench-id="xpertbench-law" title="View models and scores"><strong>XpertBench (Law)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Economics/legal</td> <td class="desc">XpertBench legal domain subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude 4.5" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-sonnet-4.5"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude Sonnet 4.5</span> </a> </td> <td data-sort="58.7"> 58.7% </td> </tr> <tr data-bench="xpertbench-research" data-open-source="0"> <td style="text-align:center" data-sort="707">707</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xpertbench-research" class="llm-fu-bench-link" data-bench-id="xpertbench-research" title="View models and scores"><strong>XpertBench (Research)</strong></a> <span class="score-symbol" title="5 scores" aria-label="5 scores">o</span> </td> <td>Economics/research</td> <td class="desc">XpertBench research domain subset.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-5" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-5-high"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-5 High</span> </a> </td> <td data-sort="48.2"> 48.2% </td> </tr> <tr data-bench="xstest" data-open-source="1"> <td style="text-align:center" data-sort="708">708</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=xstest" class="llm-fu-bench-link" data-bench-id="xstest" title="View models and scores"><strong>XSTest</strong></a> <span class="score-symbol" title="4 scores" aria-label="4 scores">o</span> </td> <td>Safety</td> <td class="desc">XSTest safety benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="IQuest Coder" data-model-version="IQuest Coder V1" data-provider="iquest"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=iquest-coder-v1-40b-thinking"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">IQuest-Coder-V1-40B-Thinking</span> </a> </td> <td data-sort="94.3"> 94.3% </td> </tr> <tr data-bench="zebralogic" data-open-source="1"> <td style="text-align:center" data-sort="709">709</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zebralogic" class="llm-fu-bench-link" data-bench-id="zebralogic" title="View models and scores"><strong>ZebraLogic</strong></a> <span class="score-symbol" title="56 scores" aria-label="56 scores">O</span> <a href="https://huggingface.co/spaces/allenai/ZeroEval" target="_blank" rel="noopener" class="lb-icon" title="Leaderboard" aria-label="Leaderboard"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <rect x="1" y="9" width="3" height="6" rx="1"></rect> <rect x="6.5" y="5" width="3" height="10" rx="1"></rect> <rect x="12" y="2" width="3" height="13" rx="1"></rect> </svg> </a> </td> <td>Logical reasoning</td> <td class="desc">Logical reasoning benchmark assessing complex pattern and rule inference.</td> <td data-sort="1"> <span class="stars" title="Usage: 2/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3 VL" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3-vl-32b-thinking"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3-VL 32B Thinking</span> </a> </td> <td data-sort="96.1"> 96.1% </td> </tr> <tr data-bench="zerobench" data-open-source="1"> <td style="text-align:center" data-sort="710">710</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zerobench" class="llm-fu-bench-link" data-bench-id="zerobench" title="View models and scores"><strong>ZeroBench</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> </td> <td>Zero-shot generalization</td> <td class="desc">Evaluates zero-shot performance across diverse tasks without task-specific finetuning.</td> <td data-sort="1"> <span class="stars" title="Usage: 1/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GLM" data-model-version="GLM 4.5V" data-provider="zai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=glm-4.5v"> <span class="brand"> <img class="logo no-invert" alt="zai logo" data-company="zai" data-country="China" src="https://furukama.com/logos/zai.svg?v=e40b4c68" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">GLM-4.5V</span> </a> </td> <td data-sort="23.4"> 23.4% </td> </tr> <tr data-bench="zerobench-sub" data-open-source="1"> <td style="text-align:center" data-sort="711">711</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zerobench-sub" class="llm-fu-bench-link" data-bench-id="zerobench-sub" title="View models and scores"><strong>ZeroBench (sub)</strong></a> <span class="score-symbol" title="33 scores" aria-label="33 scores">O</span> </td> <td>Zero-shot generalization</td> <td class="desc">Subset of ZeroBench targeting harder zero-shot reasoning cases.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Qwen" data-model-version="Qwen3.5" data-provider="qwen"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=qwen3.5-397b-a17b"> <span class="brand"> <img class="logo no-invert" alt="qwen logo" data-company="qwen" data-country="China" src="https://furukama.com/logos/qwen.svg?v=a01e6854" /> <span class="flag" role="img" aria-label="China" title="China" data-country="China">🇨🇳</span> </span> <span class="leader-name">Qwen3.5-397B-A17B</span> </a> </td> <td data-sort="41.0"> 41.0% </td> </tr> <tr data-bench="zeroscrolls-booksumsort" data-open-source="0"> <td style="text-align:center" data-sort="712">712</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-booksumsort" class="llm-fu-bench-link" data-bench-id="zeroscrolls-booksumsort" title="View models and scores"><strong>ZeroSCROLLS BookSumSort</strong></a> <span class="score-symbol" title="7 scores" aria-label="7 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context summarization</td> <td class="desc">ZeroSCROLLS split based on BookSumSort long-form summarization.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4</span> </a> </td> <td data-sort="60.5"> 60.5% </td> </tr> <tr data-bench="zeroscrolls-govreport" data-open-source="1"> <td style="text-align:center" data-sort="713">713</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-govreport" class="llm-fu-bench-link" data-bench-id="zeroscrolls-govreport" title="View models and scores"><strong>ZeroSCROLLS GovReport</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context summarization</td> <td class="desc">ZeroSCROLLS split based on the GovReport summarization benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="CoLT5" data-model-version="CoLT5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=colt5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">CoLT5</span> </a> </td> <td data-sort="41.0"> 41.0% </td> </tr> <tr data-bench="zeroscrolls-musique" data-open-source="1"> <td style="text-align:center" data-sort="714">714</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-musique" class="llm-fu-bench-link" data-bench-id="zeroscrolls-musique" title="View models and scores"><strong>ZeroSCROLLS MuSiQue</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context reasoning</td> <td class="desc">ZeroSCROLLS split derived from MuSiQue multi-hop QA.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama" data-model-version="Llama 3.3" data-provider="meta"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3.3-70b-instruct"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama 3.3 70B Instruct</span> </a> </td> <td data-sort="52.2"> 52.2% </td> </tr> <tr data-bench="zeroscrolls-narrativeqa" data-open-source="0"> <td style="text-align:center" data-sort="715">715</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-narrativeqa" class="llm-fu-bench-link" data-bench-id="zeroscrolls-narrativeqa" title="View models and scores"><strong>ZeroSCROLLS NarrativeQA</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context QA</td> <td class="desc">ZeroSCROLLS split based on the NarrativeQA reading comprehension benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Claude" data-model-version="Claude v1.3" data-provider="anthropic"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=claude-v1.3"> <span class="brand"> <img class="logo no-invert" alt="anthropic logo" data-company="anthropic" data-country="United States" src="https://furukama.com/logos/anthropic.svg?v=d5929d72" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Claude v1.3</span> </a> </td> <td data-sort="32.6"> 32.6% </td> </tr> <tr data-bench="zeroscrolls-qasper" data-open-source="1"> <td style="text-align:center" data-sort="716">716</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-qasper" class="llm-fu-bench-link" data-bench-id="zeroscrolls-qasper" title="View models and scores"><strong>ZeroSCROLLS Qasper</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context QA</td> <td class="desc">ZeroSCROLLS split based on the Qasper paper QA benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="FLAN-UL2" data-model-version="FLAN-UL2" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=flan-ul2"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">FLAN-UL2</span> </a> </td> <td data-sort="56.9"> 56.9% </td> </tr> <tr data-bench="zeroscrolls-qmsum" data-open-source="1"> <td style="text-align:center" data-sort="717">717</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-qmsum" class="llm-fu-bench-link" data-bench-id="zeroscrolls-qmsum" title="View models and scores"><strong>ZeroSCROLLS QMSum</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context summarization</td> <td class="desc">ZeroSCROLLS split based on the QMSum meeting summarization benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="CoLT5" data-model-version="CoLT5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=colt5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">CoLT5</span> </a> </td> <td data-sort="22.5"> 22.5% </td> </tr> <tr data-bench="zeroscrolls-quality" data-open-source="0"> <td style="text-align:center" data-sort="718">718</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-quality" class="llm-fu-bench-link" data-bench-id="zeroscrolls-quality" title="View models and scores"><strong>ZeroSCROLLS QuALITY</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context QA</td> <td class="desc">ZeroSCROLLS split based on the QuALITY reading comprehension benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4</span> </a> </td> <td data-sort="89.2"> 89.2% </td> </tr> <tr data-bench="zeroscrolls-spacedigest" data-open-source="1"> <td style="text-align:center" data-sort="719">719</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-spacedigest" class="llm-fu-bench-link" data-bench-id="zeroscrolls-spacedigest" title="View models and scores"><strong>ZeroSCROLLS SpaceDigest</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context summarization</td> <td class="desc">ZeroSCROLLS SpaceDigest extractive summarization task.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="Llama TFree HAT" data-model-version="Llama TFree HAT" data-provider="alephalpha"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=llama-3_1-70b-tfree-hat-sft"> <span class="brand"> <img class="logo" alt="meta logo" data-company="meta" data-country="United States" src="https://furukama.com/logos/meta.svg?v=da9bb571" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">Llama-3_1-70B-TFree-HAT-SFT</span> </a> </td> <td data-sort="77.9"> 77.9% </td> </tr> <tr data-bench="zeroscrolls-squality" data-open-source="0"> <td style="text-align:center" data-sort="720">720</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-squality" class="llm-fu-bench-link" data-bench-id="zeroscrolls-squality" title="View models and scores"><strong>ZeroSCROLLS SQuALITY</strong></a> <span class="score-symbol" title="9 scores" aria-label="9 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context summarization</td> <td class="desc">ZeroSCROLLS split based on the SQuALITY long-form summarization benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="GPT" data-model-version="GPT-4" data-provider="openai"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=gpt-4"> <span class="brand"> <img class="logo no-invert" alt="openai logo" data-company="openai" data-country="United States" src="https://furukama.com/logos/openai.svg?v=888187e8" /> <span class="flag" role="img" aria-label="United States" title="United States" data-country="United States">🇺🇸</span> </span> <span class="leader-name">GPT-4</span> </a> </td> <td data-sort="22.6"> 22.6% </td> </tr> <tr data-bench="zeroscrolls-summscreenfd" data-open-source="1"> <td style="text-align:center" data-sort="721">721</td> <td> <a href="https://furukama.com/llm-fu/benchmark/?benchmark=zeroscrolls-summscreenfd" class="llm-fu-bench-link" data-bench-id="zeroscrolls-summscreenfd" title="View models and scores"><strong>ZeroSCROLLS SummScreenFD</strong></a> <span class="score-symbol" title="8 scores" aria-label="8 scores">o</span> <a href="https://github.com/allenai/scrolls" target="_blank" rel="noopener" class="data-icon" title="Dataset" aria-label="Dataset"> <svg viewBox="0 0 16 16" fill="currentColor" aria-hidden="true"> <path d="M3 4c0-1.66 2.69-3 5-3s5 1.34 5 3-2.69 3-5 3-5-1.34-5-3zm0 3.5c0 1.66 2.69 3 5 3s5-1.34 5-3V9c0 1.66-2.69 3-5 3s-5-1.34-5-3V7.5zm0 4c0 1.66 2.69 3 5 3s5-1.34 5-3V13c0 1.66-2.69 3-5 3s-5-1.34-5-3v-1.5z"></path> </svg> </a> </td> <td>Long-context summarization</td> <td class="desc">ZeroSCROLLS split based on the SummScreenFD summarization benchmark.</td> <td data-sort="1"> <span class="stars" title="Usage: 0/39; GitHub>100: no; Recent: no; Leader: yes"> ★<span class="empty">★★★★</span> </span> </td> <td data-sort="0"></td> <td class="leader-cell" data-model-family="CoLT5" data-model-version="CoLT5" data-provider="google"> <a class="leader-link" href="https://furukama.com/llm-fu/model/?model=colt5"> <span class="brand"><span class="avatar" aria-label="generic">G</span></span> <span class="leader-name">CoLT5</span> </a> </td> <td data-sort="20.0"> 20.0% </td> </tr> </tbody> </table> </div> <div class="llm-fu-histograms" id="llm-fu-histograms"> <div class="llm-fu-histogram"> <div class="llm-fu-histogram-head"> <h3>Model Size</h3> <label class="llm-fu-log-toggle"><input type="checkbox" id="llm-fu-logsize-toggle" checked>Log scale</label> </div> <div id="llm-fu-histogram-size" class="llm-fu-histogram-chart"></div> <div class="llm-fu-histogram-footer"> <span id="llm-fu-count-size" aria-label="Model count"></span> <div id="llm-fu-legend-size" class="llm-fu-legend"></div> </div> </div> <div class="llm-fu-histogram"> <div class="llm-fu-histogram-head"> <h3>Release Date</h3> </div> <div id="llm-fu-histogram-date" class="llm-fu-histogram-chart"></div> <div class="llm-fu-histogram-footer"> <span id="llm-fu-count-date" aria-label="Model count"></span> <div id="llm-fu-legend-date" class="llm-fu-legend"></div> </div> </div> </div> </div> <script> const LLM_FU_COUNTRY_GROUPS = (() => { const EU = new Set(['AT','BE','BG','HR','CY','CZ','DK','EE','FI','FR','DE','GR','HU','IE','IT','LV','LT','LU','MT','NL','PL','PT','RO','SK','SI','ES','SE']); const EUROPE = new Set([...EU, 'GB', 'UK', 'CH', 'NO', 'IS', 'LI', 'AD', 'SM', 'VA', 'MC', 'AL', 'BA', 'ME', 'MK', 'RS', 'UA', 'BY', 'MD', 'TR', 'IC', 'GI']); const MIDDLE_EAST = new Set(['AE','SA','QA','BH','OM','KW','IL','JO','LB','SY','IQ','IR','YE','PS','TR','EG']); const EMEA = new Set([...EUROPE, ...MIDDLE_EAST]); return { EU, EUROPE, EMEA, MIDDLE_EAST }; })(); function llmFuCountryGroupSet(key){ if (!key) return null; const upper = String(key).toUpperCase(); return LLM_FU_COUNTRY_GROUPS[upper] || null; } function modelPassesFilters(modelId, rec, filters){ if (!rec) return false; const mode = (filters && filters.mode) || 'all'; const openVal = String(rec.open_source || '').toLowerCase(); if (mode === 'open') { if (!rec.open_source) return false; } else if (mode === 'openplus') { if (openVal !== 'plus') return false; } else if (mode === 'commercial') { if (rec.open_source) return false; } const mod = (filters && filters.modality) || 'all'; if (mod && mod !== 'all') { const mods = (rec.modalities || []).map(m => String(m || '').toLowerCase()); if (mod.toLowerCase() === 'text' && !mods.includes('text')) return false; if (mod.toLowerCase() === 'image' && !mods.includes('visual')) return false; } const size = String((filters && filters.size) || 'all').toUpperCase(); if (size && size !== 'ALL') { const bucket = sizeBucket(rec); if (bucket !== size) return false; } const country = normalizeCountryValue((filters && filters.country) || 'all').toUpperCase(); if (country && country !== 'ALL') { const code = normalizeCountryValue(rec.country || '').toUpperCase(); if (!code) return false; const group = llmFuCountryGroupSet(country); if (group) { if (!group.has(code)) return false; } else if (code !== country) { return false; } } const year = String((filters && filters.year) || 'all'); if (year && year !== 'all') { const launched = rec.launched || rec.release_date; if (!launched) return false; const y = new Date(launched).getUTCFullYear(); if (String(y) !== year) return false; } const provider = String((filters && filters.provider) || '').toLowerCase(); if (provider && String(rec.provider || '').toLowerCase() !== provider) return false; const family = String((filters && filters.family) || '').toLowerCase(); if (family && String(rec.model_family || '').toLowerCase() !== family) return false; const modelFilter = String((filters && filters.model) || '').toLowerCase(); if (modelFilter) { const version = String(rec.model_version || '').toLowerCase(); const base = String(rec.model || '').toLowerCase(); const idLower = String(modelId || '').toLowerCase(); if (modelFilter !== version && modelFilter !== base && modelFilter !== idLower) return false; } const searchTerm = String((filters && filters.search) || '').trim().toLowerCase(); if (searchTerm) { const haystack = [rec.model_version, rec.model_family, rec.provider, rec.model, rec.name, rec.full_name, modelId] .map(x => String(x || '').toLowerCase()) .join(' '); if (!haystack.includes(searchTerm)) return false; } return true; } let activeFilters = { mode: 'all', modality: 'all', size: 'all', country: 'all', year: 'all' }; let miniTopN = 10; function parseParamBillions(val){ if (val == null) return null; const m = String(val).match(/-?\d+(?:\.\d+)?/); return m ? parseFloat(m[0]) : null; } function sizeBucket(rec){ const sz = parseParamBillions(rec?.parameter_size); if (sz == null || !Number.isFinite(sz)) return 'Unknown'; if (sz < 2) return 'XS'; if (sz >= 2 && sz < 7) return 'S'; if (sz >= 7 && sz < 30) return 'M'; if (sz >= 30 && sz < 70) return 'L'; if (sz >= 70) return 'XL'; return 'Unknown'; } function normalizeCountryValue(val){ const v = String(val || '').trim(); if (!v) return ''; const upper = v.toUpperCase(); const map = { 'CHINA': 'CN', 'GERMANY': 'DE', 'UNITED STATES': 'US', 'USA': 'US', 'UK': 'GB', 'UNITED KINGDOM': 'GB', 'KOREA': 'KR', 'SOUTH KOREA': 'KR', 'UAE': 'AE', 'UNITED ARAB EMIRATES': 'AE', 'FRANCE': 'FR', 'CANADA': 'CA', 'SWITZERLAND': 'CH', 'POLAND': 'PL', 'JAPAN': 'JP', 'INDIA': 'IN', 'SINGAPORE': 'SG', 'GLOBAL': 'GLOBAL', }; if (map[upper]) return map[upper]; if (upper.length === 2) return upper; return upper; } function quarterLabel(ts){ const d = new Date(ts); const y = d.getUTCFullYear(); const q = Math.floor(d.getUTCMonth() / 3) + 1; return `Q${q} ${y}`; } function quarterRange(tsStart, tsEnd){ const months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']; const ds = new Date(tsStart); const de = new Date(tsEnd); const label = `${months[ds.getUTCMonth()]}–${months[Math.max(0, de.getUTCMonth()-1)]} ${ds.getUTCFullYear()}`; return label; } function monthRange(tsStart){ const months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']; const ds = new Date(tsStart); return `${months[ds.getUTCMonth()]} ${ds.getUTCFullYear()}`; } function renderHistogram(containerId, values, opts = {}){ const el = document.getElementById(containerId); if (!el) return; const series = (opts.series && opts.series.length) ? opts.series.map(s => ({ ...s, values: s.values || [] })) : [{ label: 'All', color: 'rgba(79,70,229,0.55)', values: values || [] }]; const allValues = series.flatMap(s => s.values || []); if (!allValues.length) { el.innerHTML = '<div class="llm-fu-chart-empty" style="padding:1rem;text-align:center;color:var(--color-muted)">No data</div>'; return; } const isDate = !!opts.isDate; const useLog = !!opts.logBins; const w = Math.max(el.clientWidth || 560, 360); const h = 300; const padding = { top: 24, right: 18, bottom: 32, left: 46 }; let min = Math.min(...allValues); let max = Math.max(...allValues); if (min === max) { min = min - 0.5; max = max + 0.5; } let binCount = opts.binCount || 12; let countsPerSeries = []; let totals = []; const centers = []; let binEdges = []; let domainMin = min; let domainMax = max; if (useLog) { const safeMin = Math.max(min, 1e-6); const minLog = Math.log10(safeMin); const maxLog = Math.log10(max); const step = (maxLog - minLog) / binCount; binEdges = Array.from({ length: binCount + 1 }, (_, i) => Math.pow(10, minLog + i * step)); domainMin = binEdges[0]; domainMax = binEdges[binEdges.length - 1]; countsPerSeries = series.map(() => new Array(binCount).fill(0)); totals = new Array(binCount).fill(0); for (let i = 0; i < binCount; i++) { centers.push(Math.pow(10, minLog + (i + 0.5) * step)); } series.forEach((s, si) => { (s.values || []).forEach(v => { let idx = binEdges.findIndex((edge, i) => v >= edge && v < binEdges[i + 1]); if (idx === -1) idx = binCount - 1; countsPerSeries[si][idx] += 1; totals[idx] += 1; }); }); } else if (isDate && opts.useMonths) { const start = new Date(min); start.setUTCHours(0,0,0,0); start.setUTCDate(1); const edges = []; const cursor = new Date(start.getTime()); while (cursor.getTime() <= max) { edges.push(cursor.getTime()); cursor.setUTCMonth(cursor.getUTCMonth() + 1); } edges.push(cursor.getTime()); binEdges = edges; domainMin = binEdges[0]; domainMax = binEdges[binEdges.length - 1]; binCount = Math.max(1, binEdges.length - 1); countsPerSeries = series.map(() => new Array(binCount).fill(0)); totals = new Array(binCount).fill(0); for (let i = 0; i < binEdges.length - 1; i++) { centers.push((binEdges[i] + binEdges[i + 1]) / 2); } series.forEach((s, si) => { (s.values || []).forEach(v => { let idx = binEdges.findIndex((edge, i) => v >= edge && v < binEdges[i + 1]); if (idx === -1) idx = binEdges.length - 2; countsPerSeries[si][idx] += 1; totals[idx] += 1; }); }); } else if (isDate && opts.useQuarters) { const start = new Date(min); start.setUTCHours(0,0,0,0); const startQuarter = Math.floor(start.getUTCMonth() / 3) * 3; start.setUTCMonth(startQuarter, 1); const edges = []; const cursor = new Date(start.getTime()); while (cursor.getTime() <= max) { edges.push(cursor.getTime()); cursor.setUTCMonth(cursor.getUTCMonth() + 3); } edges.push(cursor.getTime()); binEdges = edges; domainMin = binEdges[0]; domainMax = binEdges[binEdges.length - 1]; binCount = Math.max(1, binEdges.length - 1); countsPerSeries = series.map(() => new Array(binCount).fill(0)); totals = new Array(binCount).fill(0); for (let i = 0; i < binEdges.length - 1; i++) { centers.push((binEdges[i] + binEdges[i + 1]) / 2); } series.forEach((s, si) => { (s.values || []).forEach(v => { let idx = binEdges.findIndex((edge, i) => v >= edge && v < binEdges[i + 1]); if (idx === -1) idx = binEdges.length - 2; countsPerSeries[si][idx] += 1; totals[idx] += 1; }); }); } else { const binWidth = (max - min) / binCount; binEdges = Array.from({ length: binCount + 1 }, (_, i) => min + i * binWidth); domainMin = min; domainMax = max; countsPerSeries = series.map(() => new Array(binCount).fill(0)); totals = new Array(binCount).fill(0); for (let i = 0; i < binCount; i++) { centers.push(min + (i + 0.5) * binWidth); } series.forEach((s, si) => { (s.values || []).forEach(v => { let idx = Math.floor((v - min) / binWidth); if (idx < 0) idx = 0; if (idx >= binCount) idx = binCount - 1; countsPerSeries[si][idx] += 1; totals[idx] += 1; }); }); } const total = allValues.length; function estimateDensity(vals, pts, opts){ if (!vals.length || !pts.length) return []; const logMode = !!opts.log; const safeVal = (v) => Math.max(v, 1e-9); const data = logMode ? vals.map(v => Math.log10(safeVal(v))) : vals.slice(); const xs = logMode ? pts.map(v => Math.log10(safeVal(v))) : pts; const n = data.length; const mean = data.reduce((a,b)=>a+b,0) / n; const variance = data.reduce((a,v)=>a+Math.pow(v-mean,2),0) / Math.max(n-1,1); const std = Math.max(Math.sqrt(variance), 1e-6); const h = 1.06 * std * Math.pow(n, -1/5); const binWidth = Math.max(opts.binWidth || 1, 1e-6); const norm = Math.sqrt(2*Math.PI); return xs.map(x => { const sum = data.reduce((acc,v) => { const u = (x - v) / h; return acc + Math.exp(-0.5 * u * u); }, 0); const density = sum / (n * h * norm); return density * n * binWidth; }); } let densityXs = centers; if (isDate) { const startMonth = new Date(domainMin); startMonth.setUTCDate(1); startMonth.setUTCHours(0,0,0,0); const endMonth = new Date(domainMax); endMonth.setUTCDate(1); endMonth.setUTCHours(0,0,0,0); const xs = []; const cursor = new Date(startMonth.getTime()); while (cursor.getTime() <= endMonth.getTime()) { xs.push(cursor.getTime()); cursor.setUTCMonth(cursor.getUTCMonth() + 1); } densityXs = xs.length ? xs : centers; } else { if (useLog) { const logStart = Math.log10(Math.max(domainMin, 1e-9)); const logEnd = Math.log10(domainMax); const steps = 300; densityXs = Array.from({ length: steps }, (_, i) => Math.pow(10, logStart + (i / (steps - 1)) * (logEnd - logStart))); } else { const step = 1; const start = Math.max(domainMin, min); const end = domainMax; const count = Math.max(2, Math.floor((end - start) / step) + 1); densityXs = Array.from({ length: count }, (_, i) => start + i * step); } } const binWidthRaw = isDate ? (30 * 24 * 60 * 60 * 1000) : ((binEdges[1] != null) ? (useLog ? (Math.log10(binEdges[1]) - Math.log10(binEdges[0])) : (binEdges[1] - binEdges[0])) : 1); let densityCurve = estimateDensity(allValues, densityXs, { log: useLog, binWidth: binWidthRaw }); if (isDate) densityCurve = []; const maxCount = Math.max(...totals, ...(densityCurve.length ? densityCurve : [0]), 1); const yMax = maxCount; const chartW = w - padding.left - padding.right; const chartH = h - padding.top - padding.bottom; const scaleX = (v) => { if (useLog) { const minLog = Math.log10(binEdges[0]); const maxLog = Math.log10(binEdges[binEdges.length - 1]); return padding.left + ((Math.log10(v) - minLog) / (maxLog - minLog)) * chartW; } return padding.left + ((v - domainMin) / (domainMax - domainMin)) * chartW; }; const scaleY = (v) => h - padding.bottom - (v / yMax) * chartH; const gap = 0.1; let svg = `<svg viewBox="0 0 ${w} ${h}" preserveAspectRatio="none">`; svg += `<line x1="${padding.left}" y1="${h - padding.bottom}" x2="${w - padding.right}" y2="${h - padding.bottom}" stroke="currentColor" opacity="0.4"/>`; svg += `<line x1="${padding.left}" y1="${padding.top}" x2="${padding.left}" y2="${h - padding.bottom}" stroke="currentColor" opacity="0.4"/>`; totals.forEach((totalCount, i) => { const x1 = scaleX(binEdges[i]); const x2 = scaleX(binEdges[i + 1]); const fullW = Math.max(x2 - x1, 0); const width = fullW * (1 - gap); const x = x1 + (fullW - width) / 2; let stack = 0; const pctBase = total || 1; const label = isDate ? (opts.useMonths ? monthRange(binEdges[i]) : quarterRange(binEdges[i], binEdges[i + 1])) : `${binEdges[i].toFixed(0)}B–${binEdges[i+1].toFixed(0)}B`; series.forEach((s, si) => { const c = countsPerSeries[si][i]; if (!c) return; const yTop = scaleY(stack + c); const yBottom = scaleY(stack); const barH = yBottom - yTop; const pct = ((c / pctBase) * 100).toFixed(1); const title = `${s.label || 'Series'} • ${label} • ${pct}%`; const fill = s.color || 'rgba(79,70,229,0.35)'; svg += `<rect x="${x}" y="${yTop}" width="${width}" height="${barH}" fill="${fill}" stroke="${fill}"><title>${title}</title></rect>`; stack += c; }); }); if (!isDate && densityCurve.length > 1) { const densityPoints = densityCurve.map((c, i) => { const x = scaleX(densityXs[i]); const y = scaleY(c); return `${x},${y}`; }).join(' '); svg += `<polyline points="${densityPoints}" fill="none" stroke="rgba(255,255,255,0.98)" stroke-width="2.5" stroke-dasharray="3 3"/>`; } if (opts.useMonths && isDate) { const monthNames = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']; for (let i = 0; i < binCount; i++) { const center = (binEdges[i] + binEdges[i + 1]) / 2; const x = scaleX(center); const dt = new Date(binEdges[i]); const label = monthNames[dt.getUTCMonth()]; svg += `<text x="${x}" y="${h - padding.bottom + 14}" text-anchor="middle" font-size="10" fill="currentColor" opacity="0.8">${label}</text>`; } } else if (opts.useQuarters && isDate) { const start = domainMin; const end = domainMax; const ticks = []; const d = new Date(start); d.setUTCMonth(Math.floor(d.getUTCMonth() / 3) * 3, 1); d.setUTCHours(0,0,0,0); while (d.getTime() <= end) { ticks.push(d.getTime()); d.setUTCMonth(d.getUTCMonth() + 3); } ticks.push(end); ticks.forEach((t, idx) => { const dt = new Date(t); if (dt.getUTCMonth() !== 0) return; const nextEdge = (idx + 1 < ticks.length) ? ticks[idx + 1] : t; const center = (t + nextEdge) / 2; const x = scaleX(center); const label = dt.getUTCFullYear(); svg += `<text x="${x}" y="${h - padding.bottom + 14}" text-anchor="middle" font-size="11" fill="currentColor" opacity="0.8">${label}</text>`; }); } else { const tickCount = 4; for (let i = 0; i <= tickCount; i++) { const logMin = Math.log10(domainMin); const logMax = Math.log10(domainMax); const v = useLog ? Math.pow(10, logMin + (i / tickCount) * (logMax - logMin)) : (domainMin + (i / tickCount) * (domainMax - domainMin)); const x = scaleX(v); const labelVal = v; const label = isDate ? new Date(labelVal).toISOString().slice(0,10) : (opts.formatTick ? opts.formatTick(labelVal) : labelVal.toFixed(1)); svg += `<text x="${x}" y="${h - padding.bottom + 12}" text-anchor="middle" font-size="11" fill="currentColor" opacity="0.8">${label}</text>`; } } const yTicks = 4; for (let i = 0; i <= yTicks; i++) { const v = (i / yTicks) * yMax; const y = scaleY(v); svg += `<text x="${padding.left - 6}" y="${y + 4}" text-anchor="end" font-size="11" fill="currentColor" opacity="0.8">${v.toFixed(0)}</text>`; } svg += `</svg>`; el.innerHTML = svg; } function buildHistograms(filters){ const urlParams = new URLSearchParams(window.location.search); const mergedFilters = { ...(filters || {}), provider: (urlParams.get('provider') || '').toLowerCase(), family: (urlParams.get('family') || '').toLowerCase(), model: (urlParams.get('model') || '').toLowerCase(), search: (urlParams.get('q') || '').toLowerCase().trim(), year: (filters && filters.year) ? filters.year : (urlParams.get('year') || 'all'), }; const logToggle = document.getElementById('llm-fu-logsize-toggle'); const useLogBins = logToggle ? !!logToggle.checked : true; const models = window.LLM_BOB_MODELS || {}; const sizeVals = []; const dateVals = []; const sizeOpen = []; const sizeOpenPlus = []; const sizeClosed = []; const dateOpen = []; const dateOpenPlus = []; const dateClosed = []; const mergedYear = (mergedFilters && mergedFilters.year) ? String(mergedFilters.year) : 'all'; Object.entries(models).forEach(([modelId, rec]) => { if (!modelPassesFilters(modelId, rec, mergedFilters)) return; const openVal = String(rec?.open_source || '').toLowerCase(); const isOpenPlus = openVal === 'plus'; const isOpen = rec?.open_source === true; const sz = parseParamBillions(rec.parameter_size); if (sz != null && Number.isFinite(sz)) { sizeVals.push(sz); if (isOpenPlus) sizeOpenPlus.push(sz); else if (isOpen) sizeOpen.push(sz); else sizeClosed.push(sz); } const launched = rec.launched || rec.release_date; if (launched) { const t = Date.parse(launched); if (!Number.isNaN(t)) { if (mergedYear && mergedYear !== 'all') { const y = new Date(t).getUTCFullYear(); if (String(y) !== String(mergedYear)) return; } dateVals.push(t); if (isOpenPlus) dateOpenPlus.push(t); else if (isOpen) dateOpen.push(t); else dateClosed.push(t); } } }); let sizeBinCount = 25; renderHistogram('llm-fu-histogram-size', sizeVals, { binCount: sizeBinCount, logBins: useLogBins, formatTick: (v) => `${v.toFixed(0)}B`, series: [ { label: 'Open Weights', color: 'rgba(34,197,94,0.8)', values: sizeOpen }, { label: 'Open Source', color: 'rgba(134,239,172,0.85)', values: sizeOpenPlus }, { label: 'Commercial', color: 'rgba(59,130,246,0.85)', values: sizeClosed } ] }); const countSize = document.getElementById('llm-fu-count-size'); if (countSize) countSize.textContent = `${sizeVals.length} models`; const yearFilterActive = mergedYear && mergedYear !== 'all'; renderHistogram('llm-fu-histogram-date', dateVals, { binCount: 12, isDate: true, useQuarters: !yearFilterActive, useMonths: yearFilterActive, series: [ { label: 'Open Weights', color: 'rgba(34,197,94,0.8)', values: dateOpen }, { label: 'Open Source', color: 'rgba(134,239,172,0.85)', values: dateOpenPlus }, { label: 'Commercial', color: 'rgba(59,130,246,0.85)', values: dateClosed } ] }); const countDate = document.getElementById('llm-fu-count-date'); if (countDate) countDate.textContent = `${dateVals.length} models`; const mode = (mergedFilters && mergedFilters.mode) || 'all'; const legendItems = []; if (mode === 'all' || mode === 'open') { legendItems.push('<span class="llm-fu-legend-item"><span class="llm-fu-legend-swatch" style="background:rgba(34,197,94,0.8)"></span>Open Weights</span>'); legendItems.push('<span class="llm-fu-legend-item"><span class="llm-fu-legend-swatch" style="background:rgba(134,239,172,0.85)"></span>Open Source</span>'); } if (mode === 'openplus') { legendItems.push('<span class="llm-fu-legend-item"><span class="llm-fu-legend-swatch" style="background:rgba(134,239,172,0.85)"></span>Open Source</span>'); } if (mode === 'all' || mode === 'commercial') { legendItems.push('<span class="llm-fu-legend-item"><span class="llm-fu-legend-swatch" style="background:rgba(59,130,246,0.85)"></span>Commercial</span>'); } const legendHtml = legendItems.join(''); const legendSize = document.getElementById('llm-fu-legend-size'); const legendDate = document.getElementById('llm-fu-legend-date'); if (legendSize) legendSize.innerHTML = legendHtml; if (legendDate) legendDate.innerHTML = legendHtml; } (function () { const root = document.querySelector('.llm-fu'); if (root) root.style.visibility = 'hidden'; const input = document.getElementById('llm-fu-search'); const openFilter = document.getElementById('llm-fu-open-filter'); const table = document.getElementById('llm-fu-table'); const modFilter = document.getElementById('llm-fu-modality-filter'); const sizeFilter = document.getElementById('llm-fu-size-filter'); const countryFilter = document.getElementById('llm-fu-country-filter'); const yearFilter = document.getElementById('llm-fu-year-filter'); const logSizeToggle = document.getElementById('llm-fu-logsize-toggle'); if (!input || !table) return; const rows = Array.from(table.querySelectorAll('tbody tr')); input.addEventListener('input', () => applySearch()); const urlParams = new URLSearchParams(window.location.search); const modeParam = urlParams.get('models'); const modParam = urlParams.get('mod'); const sizeParam = urlParams.get('size'); const countryParamRaw = urlParams.get('country'); const countryParam = normalizeCountryValue(countryParamRaw); const yearParam = urlParams.get('year'); let pendingCountryValue = null; const qParam = urlParams.get('q') || ''; const providerParam = (urlParams.get('provider') || '').toLowerCase(); const familyParam = (urlParams.get('family') || '').toLowerCase(); const modelParam = (urlParams.get('model') || '').toLowerCase(); const scoreParam = urlParams.get('score'); const scoreMode = scoreParam === 'relative' ? 'relative' : 'absolute'; window.LLM_FU_SCORE_MODE = scoreMode; if (input && qParam) { input.value = qParam; } if (openFilter && (modeParam === 'open' || modeParam === 'openplus' || modeParam === 'all' || modeParam === 'commercial')) { openFilter.value = modeParam; } if (modFilter && (modParam === 'all' || modParam === 'text' || modParam === 'image')){ modFilter.value = modParam; } if (sizeFilter && (sizeParam === 'all' || sizeParam === 'XS' || sizeParam === 'S' || sizeParam === 'M' || sizeParam === 'L' || sizeParam === 'XL')){ sizeFilter.value = sizeParam; } if (countryFilter && countryParam) { countryFilter.value = countryParam; if (countryFilter.value !== countryParam) { pendingCountryValue = countryParam; } } if (yearFilter && yearParam) { yearFilter.value = yearParam; } openFilter?.addEventListener('change', () => { const params = new URLSearchParams(window.location.search); params.set('models', openFilter.value); if (modFilter) params.set('mod', modFilter.value); if (sizeFilter) params.set('size', sizeFilter.value); if (yearFilter) { if (yearFilter.value === 'all') params.delete('year'); else params.set('year', yearFilter.value); } syncCountryParam(params); const newUrl = window.location.pathname + '?' + params.toString() + window.location.hash; window.location.assign(newUrl); }); modFilter?.addEventListener('change', () => { const params = new URLSearchParams(window.location.search); if (openFilter) params.set('models', openFilter.value); params.set('mod', modFilter.value); if (sizeFilter) params.set('size', sizeFilter.value); if (yearFilter) { if (yearFilter.value === 'all') params.delete('year'); else params.set('year', yearFilter.value); } syncCountryParam(params); const newUrl = window.location.pathname + '?' + params.toString() + window.location.hash; window.location.assign(newUrl); }); sizeFilter?.addEventListener('change', () => { const params = new URLSearchParams(window.location.search); if (openFilter) params.set('models', openFilter.value); if (modFilter) params.set('mod', modFilter.value); params.set('size', sizeFilter.value); if (yearFilter) { if (yearFilter.value === 'all') params.delete('year'); else params.set('year', yearFilter.value); } syncCountryParam(params); const newUrl = window.location.pathname + '?' + params.toString() + window.location.hash; window.location.assign(newUrl); }); countryFilter?.addEventListener('change', () => { const params = new URLSearchParams(window.location.search); if (openFilter) params.set('models', openFilter.value); if (modFilter) params.set('mod', modFilter.value); if (sizeFilter) params.set('size', sizeFilter.value); if (yearFilter) { if (yearFilter.value === 'all') params.delete('year'); else params.set('year', yearFilter.value); } syncCountryParam(params); const newUrl = window.location.pathname + '?' + params.toString() + window.location.hash; window.location.assign(newUrl); }); yearFilter?.addEventListener('change', () => { const params = new URLSearchParams(window.location.search); if (openFilter) params.set('models', openFilter.value); if (modFilter) params.set('mod', modFilter.value); if (sizeFilter) params.set('size', sizeFilter.value); if (countryFilter) syncCountryParam(params); else params.delete('country'); if (yearFilter.value === 'all') params.delete('year'); else params.set('year', yearFilter.value); const newUrl = window.location.pathname + '?' + params.toString() + window.location.hash; window.location.assign(newUrl); }); logSizeToggle?.addEventListener('change', () => { buildHistograms(activeFilters); }); function syncCountryParam(params){ if (!countryFilter) { params.delete('country'); return; } const val = normalizeCountryValue(countryFilter.value || ''); if (!val || val === 'ALL') { params.delete('country'); } else { params.set('country', val); } } function displayCountryLabel(code){ const upper = String(code || '').toUpperCase(); if (!upper) return ''; if (upper === 'ALL') return 'All countries / regions'; if (upper === 'EU') return 'European Union (EU)'; if (upper === 'EUROPE') return 'Europe (EU + UK, CH, NO, IS)'; if (upper === 'EMEA') return 'EMEA (Europe + Middle East)'; const label = getCountryName(upper); if (label) return label; if (upper === 'GLOBAL') return 'Global'; if (/^[A-Z]{2}$/.test(upper)) return upper; return upper || code; } function initCountryFilter(initialValue){ if (!countryFilter) return; const models = window.LLM_BOB_MODELS || {}; const existing = new Set(Array.from(countryFilter.options).map(opt => String(opt.value || '').toUpperCase())); const codes = new Set(); Object.values(models).forEach(rec => { const code = String((rec && rec.country) || '').trim().toUpperCase(); if (code) codes.add(code); }); const sorted = Array.from(codes).sort((a, b) => displayCountryLabel(a).localeCompare(displayCountryLabel(b))); sorted.forEach(code => { if (existing.has(code)) return; const opt = document.createElement('option'); opt.value = code; opt.textContent = displayCountryLabel(code); countryFilter.appendChild(opt); }); Array.from(countryFilter.options).forEach(opt => { const val = String(opt.value || '').toUpperCase(); if (!val) return; opt.textContent = displayCountryLabel(val) || opt.textContent; }); if (initialValue && initialValue !== 'ALL') { const opt = countryFilter.querySelector(`option[value="${initialValue}"]`); if (opt) { countryFilter.value = initialValue; } } } function getRowSearchText(tr) { let txt = tr.innerText.toLowerCase(); const nodes = tr.querySelectorAll('[alt],[title],[aria-label],[data-company],[data-country]'); nodes.forEach(el => { const alt = (el.getAttribute('alt') || '').toLowerCase(); const title = (el.getAttribute('title') || '').toLowerCase(); const aria = (el.getAttribute('aria-label') || '').toLowerCase(); const comp = (el.getAttribute('data-company') || '').toLowerCase(); const country = (el.getAttribute('data-country') || '').toLowerCase(); txt += ` ${alt} ${title} ${aria} ${comp} ${country}`; }); return txt; } const tbody = table.querySelector('tbody'); const headers = Array.from(table.querySelectorAll('thead th')); headers.forEach((th) => th.style.cursor = 'pointer'); let sortState = { index: 0, dir: 'asc' }; headers.forEach((th, idx) => { th.addEventListener('click', () => { const type = th.getAttribute('data-type') || 'text'; const dir = (sortState.index === idx && sortState.dir === 'asc') ? 'desc' : 'asc'; const trs = Array.from(tbody.querySelectorAll('tr')); trs.sort((a, b) => compareCells(a, b, idx, type, dir)); trs.forEach(tr => tbody.appendChild(tr)); sortState = { index: idx, dir }; updateSortIndicators(); }); }); function compareCells(a, b, idx, type, dir) { const av = getVal(a, idx, type); const bv = getVal(b, idx, type); if (type === 'num') { const diff = (av - bv); return dir === 'asc' ? diff : -diff; } const cmp = String(av).localeCompare(String(bv)); return dir === 'asc' ? cmp : -cmp; } function getVal(tr, idx, type) { const td = tr.children[idx]; if (!td) return type === 'num' ? 0 : ''; const data = td.getAttribute('data-sort'); if (type === 'num') { const raw = data != null ? data : td.textContent; const m = String(raw).replace(/,/g, '').match(/-?\d+(?:\.\d+)?/); return m ? parseFloat(m[0]) : 0; } if (type === 'url') { const a = td.querySelector('a'); return a ? a.href.toLowerCase() : (data != null ? data : td.textContent).toLowerCase(); } return (data != null ? data : td.textContent).trim().toLowerCase(); } function updateSortIndicators() { headers.forEach((th, i) => { th.setAttribute('aria-sort', i === sortState.index ? (sortState.dir === 'asc' ? 'ascending' : 'descending') : 'none'); }); } function headerIndex(label, fallback) { const idx = headers.findIndex(h => h.textContent.trim().toLowerCase() === label.toLowerCase()); return idx >= 0 ? idx : fallback; } (function initialSort() { const starsIdx = headerIndex('GitHub ★', 5); const modelIdx = headerIndex('Leader', 6); const usageIdx = headerIndex('Relevance', 4); const nameIdx = headerIndex('Name', 1); const trs = Array.from(tbody.querySelectorAll('tr')); trs.sort((a, b) => { const au = getVal(a, usageIdx, 'num'); const bu = getVal(b, usageIdx, 'num'); if (au !== bu) return bu - au; const aHas = hasTopModel(a, modelIdx); const bHas = hasTopModel(b, modelIdx); if (aHas !== bHas) return bHas - aHas; const as = getVal(a, starsIdx, 'num'); const bs = getVal(b, starsIdx, 'num'); if (as !== bs) return bs - as; const an = getVal(a, nameIdx, 'text'); const bn = getVal(b, nameIdx, 'text'); return String(an).localeCompare(String(bn)); }); trs.forEach(tr => tbody.appendChild(tr)); Array.from(tbody.querySelectorAll('tr')).forEach((tr, i) => { const td = tr.children[0]; if (td) { td.textContent = String(i + 1); td.setAttribute('data-sort', String(i + 1)); td.setAttribute('data-initial-rank', String(i + 1)); } }); sortState = { index: usageIdx, dir: 'desc' }; updateSortIndicators(); })(); function hasTopModel(tr, idx) { const td = tr.children[idx]; if (!td) return 0; const txt = td.textContent.trim(); return txt ? 1 : 0; } updateSortIndicators(); window.LLM_BOB_LEADERS = [{"bench_models":5,"description":"Comprehensive QA index across diverse domains.","github":"","has_recent":0,"id":"aa-index","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AA-Index","paper":"","placements":[{"company":"xai","family":"Grok","model":"grok-4","place":1,"score":73.2,"version":"Grok 4"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":2,"score":64.3,"version":"Nemotron OpenReasoning"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":3,"score":60.9,"version":"Nemotron OpenReasoning"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":4,"score":54.7,"version":"Nemotron OpenReasoning"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":5,"score":31,"version":"Nemotron OpenReasoning"}],"stars":"","top_company":"xai","top_model":"Grok 4","top_model_family":"Grok","top_model_id":"grok-4","top_model_version":"Grok 4","top_open_source":0,"top_percent":73.2,"topic":"Multi-domain QA","total_models":1613,"url":""},{"bench_models":81,"description":"A challenging benchmark measuring language models' ability to extract, reason about, and synthesize information from long-form documents ranging from 10k to 100k tokens.","github":"","has_recent":0,"id":"aa-lcr","leaderboard":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","lower_is_better":false,"metric":"percent","name":"AA-LCR","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":76,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":75.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":75.6,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":74,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":5,"score":73,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":6,"score":72.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":72.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":8,"score":72.3,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":9,"score":71.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":71,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":11,"score":70.7,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":12,"score":70,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":13,"score":70,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":14,"score":70,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5-codex-high","place":15,"score":69,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":16,"score":69,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":17,"score":68.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":18,"score":68.7,"version":"Qwen3.5"},{"company":"xai","family":"Grok","model":"grok-4","place":19,"score":68,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":20,"score":68,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-256b","place":21,"score":67,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":22,"score":67,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-2507","place":23,"score":67,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":24,"score":67,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":25,"score":66.9,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-4","place":26,"score":66.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":27,"score":66.3,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":28,"score":66.1,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":29,"score":66,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":30,"score":66,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":31,"score":66,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":32,"score":66,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":33,"score":66,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":34,"score":65.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":35,"score":65.3,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":36,"score":65,"version":"DeepSeek V3.1"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":37,"score":65,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":38,"score":65,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":39,"score":65,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonet","place":40,"score":64.7,"version":"Claude 4"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":41,"score":64.7,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":42,"score":64.7,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":43,"score":64.3,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":44,"score":64,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":45,"score":62,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":46,"score":61.7,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":47,"score":61.7,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-3","place":48,"score":61,"version":"GPT-3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":49,"score":61,"version":"MiniMax M2"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":50,"score":60.3,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":51,"score":60,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":52,"score":59,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":53,"score":58.5,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":54,"score":55,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-preview","place":55,"score":54.7,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":56,"score":54.7,"version":"DeepSeek R1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":57,"score":54,"version":"GLM 4.6"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":58,"score":53.5,"version":"K-EXAONE"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":59,"score":53.3,"version":"DeepSeek V3"},{"company":"moonshot","family":"Kimi","model":"kimi-k20895","place":60,"score":52.3,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":61,"score":52.3,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":62,"score":52,"version":"Kimi K2"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":63,"score":51.3,"version":"Magistral 1.2"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":64,"score":51,"version":"Magistral 1.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":65,"score":51,"version":"GPT-OSS"},{"company":"openai","family":"o3","model":"gpt-o3","place":66,"score":50.7,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":67,"score":50.7,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":68,"score":50.7,"version":"GPT-OSS"},{"company":"xai","family":"Grok","model":"grok-core-poset","place":69,"score":48.3,"version":"Grok Core"},{"company":"zai","family":"GLM","model":"glm-4-flash","place":70,"score":48.3,"version":"GLM 4"},{"company":"zai","family":"GLM","model":"glm-4.5","place":71,"score":48.3,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":72,"score":48.3,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-4-memonet","place":73,"score":46,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":74,"score":46,"version":"Llama 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":75,"score":45,"version":"DeepSeek V3.1"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":76,"score":45,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":77,"score":43.7,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":78,"score":37.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":79,"score":37.3,"version":"GLM 4.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":80,"score":35.9,"version":"Nemotron 3"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":81,"score":35,"version":"Solar Open"},{"company":"meta","family":"Llama","model":"llama-4","place":82,"score":34,"version":"Llama 4"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":83,"score":34,"version":"Llama Nemotron v1.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":84,"score":34,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":85,"score":34,"version":"GPT-OSS"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":86,"score":26.33,"version":"Apriel 1.5"},{"company":"openai","family":"GPT","model":"gpt-super-105-turbo","place":87,"score":25,"version":"GPT Super 105"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":88,"score":25,"version":"GPT-5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":89,"score":20,"version":"Apriel 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":90,"score":19.7,"version":"GPT-5"},{"company":"mistral","family":"Mistral","model":"mistra-medium","place":91,"score":18.7,"version":"Mistral"},{"company":"openai","family":"GPT","model":"gpt-2","place":92,"score":18.7,"version":"GPT-2"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":93,"score":18.7,"version":"GPT-OSS"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":94,"score":16.3,"version":"Magistral 1.2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":95,"score":14,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":96,"score":14,"version":"EXAONE 4.0"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":97,"score":12,"version":"HyperCLOVA X"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":98,"score":9,"version":"Mi:dm K 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":76,"topic":"Long-context reasoning","total_models":1613,"url":"https://huggingface.co/datasets/ArtificialAnalysis/AA-LCR"},{"bench_models":25,"description":"Benchmark measuring factual recall and hallucination across economically relevant domains.","github":"","has_recent":0,"id":"aa-omniscience","leaderboard":"https://artificialanalysis.ai/evaluations/omniscience","lower_is_better":false,"metric":"index","name":"AA-Omniscience","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":1,"score":13,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":10,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":10,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":4,"score":2,"version":"GPT-5.1"},{"company":"xai","family":"Grok","model":"grok-4","place":5,"score":1,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":6,"score":-2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":7,"score":-4,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":8,"score":-23,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":9,"score":-23,"version":"Kimi K2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":10,"score":-30,"version":"MiniMax M2.1"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":11,"score":-31,"version":"Grok 4.1"},{"company":"kuaishou","family":"KAT","model":"kat-coder-pro-v1","place":12,"score":-36,"version":"KAT-Coder Pro v1"},{"company":"zai","family":"GLM","model":"glm-4.7","place":13,"score":-36,"version":"GLM 4.7"},{"company":"mistral","family":"Mistral","model":"mistral-large-3","place":14,"score":-41,"version":"Mistral 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":15,"score":-42,"version":"MiMo V2"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":16,"score":-43,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":17,"score":-48,"version":"Qwen3"},{"company":"amazon","family":"Nova","model":"nova-pro","place":18,"score":-50,"version":"Nova"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":19,"score":-52,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":20,"score":-52,"version":"Nemotron 3"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":21,"score":-55,"version":"Mi:dm K 2.5"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":22,"score":-57,"version":"K2 V2"},{"company":"amazon","family":"Nova","model":"nova-lite","place":23,"score":-58,"version":"Nova"},{"company":"servicenow","family":"Apriel","model":"apriel-1.6-15b-thinker","place":24,"score":-60,"version":"Apriel 1.6"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":25,"score":-65,"version":"GPT-OSS"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro Preview","top_model_family":"Gemini","top_model_id":"gemini-3-pro-preview","top_model_version":"Gemini 3","top_open_source":0,"top_percent":13,"topic":"Knowledge and hallucination","total_models":1613,"url":"https://artificialanalysis.ai/evaluations/omniscience"},{"bench_models":10,"description":"Industry-focused benchmark assessing domain QA and reasoning.","github":"","has_recent":0,"id":"acebench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AceBench","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":1,"score":82.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":2,"score":80.1,"version":"GPT-4.1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":3,"score":79.6,"version":"openPangu-R"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":4,"score":76.5,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":5,"score":76.2,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":6,"score":75.6,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":7,"score":74.5,"version":"Gemini 2.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":8,"score":74.3,"version":"openPangu-R"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":9,"score":72.7,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":10,"score":70.5,"version":"Qwen3"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2","top_model_family":"Kimi","top_model_id":"kimi-k2","top_model_version":"Kimi K2","top_open_source":1,"top_percent":82.2,"topic":"Industry QA","total_models":1613,"url":""},{"bench_models":23,"description":"Safety and behavior evaluation with yes/no questions.","github":"https://github.com/ibm/ACPBench","has_recent":0,"id":"acp-bench-bool","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ACP-Bench Bool","paper":"https://arxiv.org/abs/2410.05669","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":1,"score":85.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":2,"score":81.84,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":3,"score":80.49,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":4,"score":78.71,"version":"Llama 3"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":5,"score":77.74,"version":"GPT-4o"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":6,"score":67.4,"version":"Codestral"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":7,"score":66.67,"version":"Llama 3.1"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":8,"score":65.53,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":9,"score":61.53,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-c-70b","place":10,"score":59.09,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-c-34b","place":11,"score":59.02,"version":"Llama 1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-33b","place":12,"score":57.58,"version":"DeepSeek"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":13,"score":55.72,"version":"Llama 2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":14,"score":55.63,"version":"Mixtral"},{"company":"microsoft","family":"Phi","model":"phi-3.128k","place":15,"score":55.53,"version":"Phi 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-instruct-7b","place":16,"score":55.45,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":17,"score":55,"version":"Mistral"},{"company":"IBM","family":"Granite","model":"granite-c-8b","place":18,"score":53.09,"version":"Granite 3.1"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":19,"score":51.8,"version":"gemma-7b"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":20,"score":51.46,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-c-instruct-8b","place":21,"score":50.53,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-7b","place":22,"score":48.2,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-13b","place":23,"score":47.79,"version":"Granite 3.1"}],"stars":"","top_company":"qwen","top_model":"Qwen3-32B","top_model_family":"Qwen","top_model_id":"qwen3-32b","top_model_version":"Qwen3","top_open_source":1,"top_percent":85.1,"topic":"Safety evaluation (boolean)","total_models":1613,"url":"https://ibm.github.io/ACPBench/"},{"bench_models":23,"description":"Safety and behavior evaluation with multiple-choice questions.","github":"https://github.com/ibm/ACPBench","has_recent":0,"id":"acp-bench-mcq","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ACP-Bench MCQ","paper":"https://arxiv.org/abs/2410.05669","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":1,"score":82.1,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":2,"score":77.42,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":3,"score":74.97,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":4,"score":74.3,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":5,"score":66.94,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":6,"score":56.5,"version":"GPT-4o"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":7,"score":51.44,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":8,"score":44.05,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":9,"score":41.52,"version":"Llama 3.1"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":10,"score":40.97,"version":"Codestral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":11,"score":39.25,"version":"Mixtral"},{"company":"mistral","family":"Mistral","model":"mistral-instruct-7b","place":12,"score":37.3,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-c-34b","place":13,"score":35.71,"version":"Llama 1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-33b","place":14,"score":35.11,"version":"DeepSeek"},{"company":"microsoft","family":"Phi","model":"phi-3.128k","place":15,"score":34.75,"version":"Phi 3.5"},{"company":"meta","family":"Llama","model":"llama-c-70b","place":16,"score":32.87,"version":"Llama 1"},{"company":"IBM","family":"Granite","model":"granite-c-instruct-8b","place":17,"score":32.63,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":18,"score":29.71,"version":"Llama 2"},{"company":"IBM","family":"Granite","model":"granite-7b","place":19,"score":29.67,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-c-8b","place":20,"score":29.21,"version":"Granite 3.1"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":21,"score":28.93,"version":"gemma-7b"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":22,"score":28.67,"version":"Mistral"},{"company":"IBM","family":"Granite","model":"granite-13b","place":23,"score":26.66,"version":"Granite 3.1"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B","top_model_family":"Llama","top_model_id":"llama-3.3-70b","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":82.1,"topic":"Safety evaluation (MCQ)","total_models":1613,"url":"https://ibm.github.io/ACPBench/"},{"bench_models":12,"description":"Code generation benchmark for diverse coding tasks.","github":"","has_recent":0,"id":"aethercode","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AetherCode","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":73.8,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":2,"score":60.6,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":57.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":56.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":5,"score":56.1,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":6,"score":43.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":7,"score":42.6,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":8,"score":41.5,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":9,"score":38.2,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":32.4,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":31.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":12,"score":29.8,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":18.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":16.4,"version":"Claude 4.5"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":73.8,"topic":"Code generation","total_models":1613,"url":""},{"bench_models":7,"description":"Company-level agent reasoning and decision-making benchmark.","github":"","has_recent":0,"id":"agentcompany","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AgentCompany","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":1,"score":41,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":39.3,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":3,"score":37,"version":"Claude 4"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":4,"score":36,"version":"MiniMax M2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":5,"score":35,"version":"GLM 4.6"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":6,"score":34,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":7,"score":30,"version":"Kimi K2"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.5","top_model_family":"Claude","top_model_id":"claude-4.5-sonnet","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":41,"topic":"Agent reasoning","total_models":1613,"url":""},{"bench_models":21,"description":"Interactive evaluation suite for autonomous agents across tools and tasks.","github":"https://github.com/ethz-spylab/agentdojo","has_recent":0,"id":"agentdojo","leaderboard":"https://agentdojo.spylab.ai/results/","lower_is_better":false,"metric":"percent","name":"AgentDojo","paper":"https://arxiv.org/abs/2406.13352","placements":[{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":1,"score":88.66,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":2,"score":88.66,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":3,"score":79.38,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20240620","place":4,"score":79.38,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-05-13","place":5,"score":69.07,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":6,"score":68.04,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3-opus-20240229","place":7,"score":68.04,"version":"Claude 3"},{"company":"openai","family":"GPT","model":"gpt-4-0125-preview","place":8,"score":65.98,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":9,"score":64.95,"version":"GPT-4"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":10,"score":61.86,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet-20240229","place":11,"score":53.61,"version":"Claude 3"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-exp","place":12,"score":46.39,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-001","place":13,"score":46.39,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":14,"score":43.3,"version":"Gemini 2.0"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku-20240307","place":15,"score":39.18,"version":"Claude 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":16,"score":38.14,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-001","place":17,"score":38.14,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0125","place":18,"score":35.05,"version":"GPT-3.5"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":19,"score":34.02,"version":"Llama 3"},{"company":"cohere","family":"Command","model":"command-r","place":20,"score":26.8,"version":"Command"},{"company":"cohere","family":"Command","model":"command-r-plus","place":21,"score":24.74,"version":"Command"}],"stars":"","top_company":"anthropic","top_model":"Claude 3.7 Sonnet","top_model_family":"Claude","top_model_id":"claude-3.7-sonnet","top_model_version":"Claude 3.7","top_open_source":0,"top_percent":88.66,"topic":"Agent evaluation","total_models":1613,"url":"https://agentdojo.spylab.ai/"},{"bench_models":22,"description":"Agentic coding benchmark for autonomous software tasks.","github":"","has_recent":0,"id":"agentic-coding","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Agentic Coding","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":1,"score":53.8,"version":"Gemini 3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":2,"score":52.5,"version":"IQuest Coder V1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":3,"score":51.3,"version":"IQuest Coder V1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":51,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":47.5,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":6,"score":47.1,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":7,"score":46.3,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":8,"score":44.5,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":9,"score":37.5,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":10,"score":36.3,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":11,"score":35,"version":"GPT-5.1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":12,"score":23.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":13,"score":23.8,"version":"DeepSeek V3.2"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":14,"score":21.3,"version":"KAT"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":15,"score":17.5,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":16,"score":15,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":17,"score":8.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":18,"score":8.8,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":19,"score":7.5,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":20,"score":6.3,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":21,"score":5,"version":"DeepSeek Coder V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":22,"score":5,"version":"Qwen2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Flash Preview","top_model_family":"Gemini","top_model_id":"gemini-3-flash-preview","top_model_version":"Gemini 3","top_open_source":0,"top_percent":53.8,"topic":"Agentic coding","total_models":1613,"url":""},{"bench_models":26,"description":"English subset of AGIEval; academic and professional exam questions.","github":"","has_recent":0,"id":"agieval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AGIEval (English)","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":1,"score":92.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":2,"score":90,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":3,"score":88.2,"version":"OLMo 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":4,"score":88.1,"version":"DeepSeek R1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":5,"score":87.8,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":6,"score":85.9,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":7,"score":78.9,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":8,"score":76.9,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":9,"score":71.6,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":10,"score":71.6,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":11,"score":70.9,"version":"Gemma 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":12,"score":70.63,"version":"Granite 4.0"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":13,"score":68.4,"version":"OLMo 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":14,"score":68.32,"version":"Nemotron 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":15,"score":64.6,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":16,"score":64.29,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":17,"score":63,"version":"Llama 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":18,"score":62.15,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":19,"score":59,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":20,"score":47.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":21,"score":47.1,"version":"Llama 3"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":22,"score":41.7,"version":"Gemma 2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":23,"score":30.15,"version":"SmollM3"},{"company":"arcee","family":"AFM","model":"afm-4.5b","place":24,"score":30.04,"version":"AFM"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":25,"score":28.13,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":26,"score":27.6,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":27,"score":24.2,"version":"Gemma 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL 32B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-32b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":92.2,"topic":"Exams","total_models":1613,"url":""},{"bench_models":6,"description":"LSAT Analytical Reasoning subset from AGIEval benchmark.","github":"","has_recent":0,"id":"agieval-lsat-ar","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AGIEval LSAT-AR","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":1,"score":30.4,"version":"Qwen2.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":2,"score":28.7,"version":"Nemotron Nano v2"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":3,"score":24.8,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":4,"score":23.4,"version":"Marin 32B"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":5,"score":22.6,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":6,"score":22.2,"version":"Gemma 3"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5 32B Base","top_model_family":"Qwen","top_model_id":"qwen2.5-32b-base","top_model_version":"Qwen2.5","top_open_source":1,"top_percent":30.4,"topic":"Law exam reasoning","total_models":1613,"url":""},{"bench_models":123,"description":"Visual question answering over science and diagram images.","github":"","has_recent":0,"id":"ai2d","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AI2D","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":98.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":97.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":96.4,"version":"Gemini 2.5"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":4,"score":96.3,"version":"Molmo"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":5,"score":95.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":95.8,"version":"GPT-5"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":7,"score":95.8,"version":"Molmo2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":8,"score":95.6,"version":"GPT-5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":9,"score":95.6,"version":"Molmo2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":10,"score":95.3,"version":"ERNIE 4.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":11,"score":94.7,"version":"Claude 3.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":12,"score":94.4,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":94.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":14,"score":94.2,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":94.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":16,"score":93.9,"version":"Qwen3.5"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":17,"score":93.8,"version":"Pixtral"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":18,"score":93.8,"version":"Pixtral"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":19,"score":93.72,"version":"Mistral 3.1"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":20,"score":93.7,"version":"Mistral 3"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":21,"score":93.7,"version":"Molmo2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":22,"score":93.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":23,"score":93.3,"version":"Qwen3.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":24,"score":93.2,"version":"Molmo"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":25,"score":92.91,"version":"Mistral 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":26,"score":92.9,"version":"Qwen3.5"},{"company":"unknown","family":"PLM","model":"plm-8b","place":27,"score":92.7,"version":"PLM"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":28,"score":92.6,"version":"Qwen3.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":29,"score":92.3,"version":"Llama 3.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":30,"score":92.2,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":31,"score":91.7,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":32,"score":91.5,"version":"Claude 4.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":33,"score":91.1,"version":"Llama 3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":34,"score":91.03,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":35,"score":90.9,"version":"Gemini 2.5"},{"company":"unknown","family":"PLM","model":"plm-3b","place":36,"score":90.9,"version":"PLM"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":37,"score":90.8,"version":"Kimi K2.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":38,"score":90.7,"version":"Molmo"},{"company":"openai","family":"GPT","model":"gpt-5","place":39,"score":89.7,"version":"GPT-5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":40,"score":89.6,"version":"Ovis 2.6"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":41,"score":89.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":42,"score":89.5,"version":"Qwen3 VL"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":43,"score":89.5,"version":"Keye-VL 1.5"},{"company":"openai","family":"GPT","model":"gpt-4v","place":44,"score":89.4,"version":"GPT-4V"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":45,"score":89.35,"version":"Step3-VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":46,"score":89.2,"version":"Qwen3 VL"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":47,"score":89.2,"version":"Bagel"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":48,"score":89.2,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":49,"score":89.2,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":50,"score":89.1,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":51,"score":89.05,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":52,"score":89,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":53,"score":88.93,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":54,"score":88.8,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":55,"score":88.7,"version":"Qwen2.5 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":56,"score":88.4,"version":"dots.vlm1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":57,"score":88.4,"version":"Gemini 2.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":58,"score":88.37,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":59,"score":88.3,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":60,"score":88.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":61,"score":88.1,"version":"Claude 3"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":62,"score":88.1,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":63,"score":88.1,"version":"Qwen2 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":64,"score":87.9,"version":"GLM 4.1V"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":65,"score":87.9,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":66,"score":87.7,"version":"Claude 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":67,"score":87.6,"version":"InternVL 2"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":68,"score":87.3,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":69,"score":87,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":70,"score":86.9,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":71,"score":86.7,"version":"Claude 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":72,"score":86.7,"version":"GPT-5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":73,"score":86.5,"version":"MiniCPM V 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":74,"score":86.4,"version":"Claude 4.1"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":75,"score":86.4,"version":"MolmoE"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":76,"score":86.4,"version":"InternVL 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":77,"score":86.2,"version":"GPT-5"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":78,"score":86,"version":"Manzano"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":79,"score":85.7,"version":"Qwen3 VL"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":80,"score":85.6,"version":"LLaVA OneVision"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":81,"score":85.2,"version":"InternVL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":82,"score":85,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":83,"score":84.96,"version":"MiMo VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":84,"score":84.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":85,"score":84.6,"version":"GPT-4o"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":86,"score":84.5,"version":"Gemma 3"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":87,"score":84.5,"version":"Eagle 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":88,"score":84.1,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":89,"score":84,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":90,"score":83.9,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":91,"score":83.8,"version":"InternVL 2"},{"company":"stepfun","family":"Step","model":"step-3","place":92,"score":83.7,"version":"Step 3"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":93,"score":83.6,"version":"FastVLM"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":94,"score":83.5,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":95,"score":83.32,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":96,"score":83.2,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":97,"score":83,"version":"Qwen2 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":98,"score":82.6,"version":"GPT-4o"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":99,"score":82.6,"version":"InternVL 3.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":100,"score":82.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":101,"score":82.4,"version":"Claude 4"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":102,"score":82.34,"version":"InternVL 3.5"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":103,"score":82.2,"version":"Manzano"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":104,"score":82,"version":"jina-VLM"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":105,"score":81.9,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":106,"score":81.6,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":107,"score":81.4,"version":"Claude 3.7"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":108,"score":81.4,"version":"LLaVA OneVision"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":109,"score":81.4,"version":"InternVL 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":110,"score":80.7,"version":"GPT-5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":111,"score":80.2,"version":"Gemma 3"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":112,"score":79.7,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":113,"score":79.7,"version":"Cambrian"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":114,"score":79.1,"version":"Gemini 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":115,"score":78.8,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":116,"score":78.7,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":117,"score":78.6,"version":"InternVL 3"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":118,"score":78.1,"version":"Phi 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":119,"score":77.4,"version":"FastVLM"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":120,"score":77.2,"version":"MM1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":121,"score":76.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":122,"score":76.8,"version":"Qwen2.5 Omni"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":123,"score":76.7,"version":"Phi-3"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":124,"score":74.9,"version":"InternVL 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":125,"score":74.7,"version":"Qwen2 VL"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":126,"score":74.2,"version":"xGen-MM"},{"company":"apple","family":"MM1","model":"mm1-30b","place":127,"score":73.3,"version":"MM1"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":128,"score":73,"version":"Cambrian 1"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":129,"score":72.3,"version":"PaliGemma"},{"company":"baai","family":"Emu","model":"emu3-8b","place":130,"score":70,"version":"Emu3"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":131,"score":68.1,"version":"Janus Pro"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":132,"score":68,"version":"FastVLM"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":133,"score":67,"version":"Janus Pro"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":134,"score":65.7,"version":"MM1.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":135,"score":65.7,"version":"GPT-5"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":136,"score":62.9,"version":"MiniCPM V 2.0"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":137,"score":61.1,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":138,"score":55.5,"version":"LLaVA 1.5"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":139,"score":51,"version":"Gemini Nano"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":140,"score":34.9,"version":"MolmoE"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":141,"score":0,"version":"VILA 1.5"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":142,"score":0,"version":"BLIP-3"},{"company":"bytedance","family":"LLaVA","model":"llava-next-34b","place":143,"score":0,"version":"LLaVA-NeXT"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":144,"score":0,"version":"BLIP-3O"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":98.7,"topic":"Diagram understanding (VQA)","total_models":1613,"url":""},{"bench_models":20,"description":"Non-agentic code generation benchmark from AICodeKing.","github":"","has_recent":0,"id":"aicodeking-non-agentic","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AICodeKing Non-Agentic","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":100,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":100,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-5","place":3,"score":100,"version":"GLM 5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-max","place":4,"score":74,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":5,"score":65,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":64,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":7,"score":43,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":8,"score":43,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":9,"score":43,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":42,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":11,"score":42,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":12,"score":41,"version":"DeepSeek V3.2"},{"company":"","family":"Sonoma","model":"sonoma-sky","place":13,"score":41,"version":"Sonoma"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":14,"score":40,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":15,"score":40,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":16,"score":40,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":17,"score":40,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":18,"score":38,"version":"Grok Code"},{"company":"openai","family":"GPT","model":"gpt-5.1-codex-mini-high","place":19,"score":36,"version":"GPT-5.1 Codex"},{"company":"","family":"Sonic","model":"sonic","place":20,"score":36,"version":"Sonic"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":100,"topic":"Code generation (non-agentic)","total_models":1613,"url":""},{"bench_models":10,"description":"Measures interactive code editing quality within the Aider assistant workflow.","github":"","has_recent":0,"id":"aider-code-editing","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Aider Code Editing","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":89.85,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":2,"score":88.16,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":3,"score":85.34,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":84.4,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":5,"score":83.65,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":6,"score":55.64,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":7,"score":49.62,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":8,"score":45.11,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":9,"score":43.61,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":10,"score":39.85,"version":"LLaDA 2.0"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":89.85,"topic":"Code editing","total_models":1613,"url":"https://aider.chat/docs/leaderboards/"},{"bench_models":108,"description":"Aider polyglot coding leaderboard.","github":"","has_recent":0,"id":"aider-polyglot","leaderboard":"https://aider.chat/docs/leaderboards/","lower_is_better":false,"metric":"percent","name":"Aider-Polyglot","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":1,"score":92.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":88,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":88,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":87.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-medium","place":5,"score":86.7,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-pro","place":6,"score":84.9,"version":"o3"},{"company":"openai","family":"o3","model":"o3-pro-high","place":7,"score":84.9,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":8,"score":83.1,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":9,"score":81.3,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-low","place":10,"score":81.3,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-high","place":11,"score":81.3,"version":"o3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":12,"score":80.8,"version":"IQuest Coder V1"},{"company":"xai","family":"Grok","model":"grok-4","place":13,"score":79.6,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-high","place":14,"score":79.6,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":15,"score":79.1,"version":"Gemini 2.5"},{"company":"ensemble","family":"Composite","model":"o3-high-plus-gpt-4.1","place":16,"score":78.2,"version":"Composite"},{"company":"openai","family":"o3","model":"o3","place":17,"score":76.9,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-05-06","place":18,"score":76.9,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":19,"score":76.3,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":20,"score":76.1,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":21,"score":74.5,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":22,"score":72.9,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-03-25","place":23,"score":72.9,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":24,"score":72,"version":"Claude 4"},{"company":"openai","family":"o4","model":"o4-mini","place":25,"score":72,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":26,"score":72,"version":"Claude 4"},{"company":"openai","family":"o4","model":"o4-mini-high","place":27,"score":72,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0528","place":28,"score":71.6,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":29,"score":71.4,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":30,"score":70.7,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":31,"score":70.7,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":32,"score":69.9,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-next","place":33,"score":66.2,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":34,"score":65.3,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":35,"score":64.9,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":36,"score":64.9,"version":"Claude 3.7"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t2-chimera","place":37,"score":64.4,"version":"DeepSeek-TNG R1T2"},{"company":"ensemble","family":"Composite","model":"deepseek-r1-plus-claude-3-5-sonnet-20241022","place":38,"score":64,"version":"Composite"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":39,"score":62.3,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":40,"score":61.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":41,"score":61.8,"version":"Qwen3"},{"company":"openai","family":"o1","model":"o1","place":42,"score":61.7,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":43,"score":61.7,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17-high","place":44,"score":61.7,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":45,"score":61.3,"version":"Claude 4"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":46,"score":61,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":47,"score":60.4,"version":"Claude 3.7"},{"company":"openai","family":"o3","model":"o3-mini-high","place":48,"score":60.4,"version":"o3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":49,"score":60,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":50,"score":59.6,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":51,"score":59.1,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":52,"score":57.3,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":53,"score":56.9,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":54,"score":56.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":55,"score":56.4,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":56,"score":55.1,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":57,"score":55.1,"version":"DeepSeek V3"},{"company":"quasar","family":"Quasar","model":"quasar-alpha","place":58,"score":54.7,"version":"Quasar"},{"company":"openai","family":"o3","model":"o3-mini","place":59,"score":53.8,"version":"o3"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":60,"score":53.8,"version":"o3"},{"company":"xai","family":"Grok","model":"grok-3-beta-think","place":61,"score":53.3,"version":"Grok 3"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":62,"score":53.3,"version":"Grok 3"},{"company":"openbmb","family":"Optimus","model":"optimus-alpha","place":63,"score":52.9,"version":"Optimus"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":64,"score":52.4,"version":"GPT-4.1"},{"company":"zai","family":"GLM","model":"glm-4.7","place":65,"score":52.1,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20241022","place":66,"score":51.6,"version":"Claude 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":67,"score":50.7,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":68,"score":50.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":69,"score":49.8,"version":"Qwen3 Next"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta-high","place":70,"score":49.3,"version":"Grok 3"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t-chimera","place":71,"score":48.4,"version":"DeepSeek-TNG R1T"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-chat-v3-prev","place":72,"score":48.4,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-04-17","place":73,"score":47.1,"version":"Gemini 2.5"},{"company":"magistral","family":"Magistral","model":"magistral-medium","place":74,"score":47.1,"version":"Magistral"},{"company":"openai","family":"GPT","model":"chatgpt-4o-latest-20250329","place":75,"score":45.3,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4.5-preview","place":76,"score":44.9,"version":"GPT-4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":77,"score":44,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":78,"score":44,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":79,"score":41.8,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":80,"score":40,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":81,"score":40,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-exp-1206","place":82,"score":38.2,"version":"Gemini 1.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":83,"score":35.6,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro-exp-02-05","place":84,"score":35.6,"version":"Gemini 2.0"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta-low","place":85,"score":34.7,"version":"Grok 3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":86,"score":34.2,"version":"KAT"},{"company":"openai","family":"o1","model":"o1-mini-2024-09-12","place":87,"score":32.9,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":88,"score":32.4,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":89,"score":29.3,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":90,"score":28.9,"version":"Mistral 3"},{"company":"anthropic","family":"Claude","model":"claude-3-5-haiku-20241022","place":91,"score":28,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"chatgpt-4o-latest-20250215","place":92,"score":27.1,"version":"GPT-4o"},{"company":"ensemble","family":"Composite","model":"qwq-32b-plus-qwen2.5-coder-32b-instruct","place":93,"score":26.2,"version":"Composite"},{"company":"openai","family":"GPT","model":"gpt-4o","place":94,"score":25.8,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":95,"score":23.1,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-exp","place":96,"score":22.2,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen-max-2025-01-25","place":97,"score":21.8,"version":"Qwen Max"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":98,"score":20.9,"version":"QwQ"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":99,"score":20,"version":"Kimi Dev"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-thinking-exp-01-21","place":100,"score":18.2,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":101,"score":18.2,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-chat-v2.5","place":102,"score":17.8,"version":"DeepSeek V2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":103,"score":16.4,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":104,"score":15.6,"version":"Llama 4"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":105,"score":15.6,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":106,"score":14.7,"version":"Qwen2.5"},{"company":"01.ai","family":"Yi","model":"yi-lightning","place":107,"score":12.9,"version":"Yi"},{"company":"cohere","family":"Command","model":"command-a-03-2025-quality","place":108,"score":12,"version":"Command"},{"company":"mistral","family":"Codestral","model":"codestral-2501","place":109,"score":11.1,"version":"Codestral"},{"company":"openhands","family":"OpenHands","model":"openhands-lm-32b-v0.1","place":110,"score":10.2,"version":"OpenHands"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":111,"score":8.9,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct-hyperbolic","place":112,"score":8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":113,"score":8,"version":"Qwen2.5"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":114,"score":5.3,"version":"Seed Coder"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":115,"score":4.9,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":116,"score":4.9,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":117,"score":3.6,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":118,"score":2.2,"version":"DeepSeek Coder V2"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro Preview","top_model_family":"Gemini","top_model_id":"gemini-3-pro-preview","top_model_version":"Gemini 3","top_open_source":0,"top_percent":92.9,"topic":"Code assistant eval","total_models":1613,"url":"https://aider.chat/docs/leaderboards/"},{"bench_models":19,"description":"Aider polyglot leaderboard using diff mode (pass@2).","github":"","has_recent":0,"id":"aider-polyglot-diff","leaderboard":"https://aider.chat/docs/leaderboards/","lower_is_better":false,"metric":"pass@2","name":"Aider-Polyglot (Diff)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":1,"score":91.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":89.4,"version":"Claude 4.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":3,"score":82.7,"version":"IQuest Coder V1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":78.8,"version":"Claude 4.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":5,"score":68.9,"version":"IQuest Coder V1"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":6,"score":63.1,"version":"GPT-5.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":7,"score":60,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":8,"score":57.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":9,"score":53.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":10,"score":28.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":11,"score":25.3,"version":"Qwen3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":12,"score":16.4,"version":"KAT"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":13,"score":12,"version":"Kimi Dev"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":14,"score":8.9,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":15,"score":8.4,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":16,"score":8,"version":"Qwen2.5"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":17,"score":6.2,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":18,"score":1.8,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":19,"score":1.3,"version":"DeepSeek Coder V2"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro Preview","top_model_family":"Gemini","top_model_id":"gemini-3-pro-preview","top_model_version":"Gemini 3","top_open_source":0,"top_percent":91.9,"topic":"Code assistant eval","total_models":1613,"url":"https://aider.chat/docs/leaderboards/"},{"bench_models":171,"description":"American Invitational Mathematics Examination 2024 problems.","github":"","has_recent":0,"id":"aime-2024","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AIME 2024","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":1,"score":96.6,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":2,"score":96,"version":"GPT-OSS"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":3,"score":95.8,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":94.78,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":5,"score":94.3,"version":"GPT-OSS"},{"company":"openai","family":"o4","model":"o4-mini","place":6,"score":93.4,"version":"o4"},{"company":"openai","family":"o3","model":"o3-high","place":7,"score":92.26,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":92,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":9,"score":91.87,"version":"DeepSeek V3.1"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":10,"score":91.82,"version":"Magistral 1.2"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":11,"score":91.7,"version":"Solar Open"},{"company":"openai","family":"o3","model":"o3","place":12,"score":91.6,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0528","place":13,"score":91.4,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":14,"score":91.4,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":15,"score":91,"version":"Qwen3"},{"company":"primeintellect","family":"INTELLECT","model":"intellect-3","place":16,"score":90.8,"version":"INTELLECT"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":17,"score":90.4,"version":"Nanbeige4"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":18,"score":89.7,"version":"Nemotron Cascade"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":19,"score":89.58,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":20,"score":89.5,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":21,"score":89.2,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":22,"score":89.2,"version":"Qwen3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":23,"score":89,"version":"openPangu-R"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":24,"score":88.8,"version":"Nemotron Cascade"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":25,"score":88.7,"version":"GLM 4.5"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":26,"score":88.3,"version":"Motif 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":27,"score":88.1,"version":"DeepSeek V3.2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":28,"score":88.1,"version":"Falcon H1R"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":29,"score":87.8,"version":"Nemotron OpenReasoning"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":30,"score":87.09,"version":"Nemotron"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":31,"score":86.7,"version":"Seed 1.5"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":32,"score":86.67,"version":"Falcon H1R"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":33,"score":86.3,"version":"Qwen3 VL"},{"company":"mbzuai","family":"K2","model":"k2-think","place":34,"score":86.26,"version":"K2-THINK"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":35,"score":86.2,"version":"Apriel 1.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":36,"score":86.14,"version":"Magistral 1.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":37,"score":86,"version":"MiniMax M1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":38,"score":85.8,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":39,"score":85.7,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":40,"score":84.7,"version":"Nemotron OpenReasoning"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":41,"score":84.6,"version":"GLM 4.5"},{"company":"antgroup","family":"Ring","model":"ring-mini-sparse-2.0-exp","place":42,"score":84.58,"version":"Ring 2.0"},{"company":"xai","family":"Grok","model":"grok-3-beta-think","place":43,"score":83.9,"version":"Grok 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":44,"score":83.75,"version":"MiniCPM SALA"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":45,"score":83.3,"version":"MiniMax M1"},{"company":"openai","family":"o1","model":"o1","place":46,"score":83.3,"version":"o1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1","place":47,"score":83.3,"version":"MiniMax M1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":48,"score":83.3,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":49,"score":83.3,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":50,"score":83.3,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":51,"score":83.2,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":52,"score":82.7,"version":"Qwen3"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t2-chimera","place":53,"score":82.3,"version":"DeepSeek-TNG R1T2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":54,"score":82.3,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":55,"score":81.9,"version":"Nemotron Nano v2"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":56,"score":81.46,"version":"Ministral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":57,"score":81.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":58,"score":81.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":59,"score":81.35,"version":"Qwen3 Next"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":60,"score":80.83,"version":"MiniCPM 4.1"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-ultra-253b-v1","place":61,"score":80.8,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":62,"score":80.8,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":63,"score":80.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":64,"score":80.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":65,"score":80.4,"version":"Qwen3"},{"company":"weibo","family":"VibeThinker","model":"vibethinker-1.5b","place":66,"score":80.3,"version":"VibeThinker"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":67,"score":80,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":68,"score":80,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":69,"score":79.8,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0120","place":70,"score":79.8,"version":"0120"},{"company":"openai","family":"o3","model":"o3-mini","place":71,"score":79.6,"version":"o3"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":72,"score":79.6,"version":"o3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":73,"score":79.5,"version":"QwQ"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":74,"score":79.4,"version":"Qwen3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":75,"score":79.38,"version":"QwQ"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":76,"score":79.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":77,"score":79.3,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":78,"score":79,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":79,"score":78.3,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":80,"score":77.9,"version":"Qwen3"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":81,"score":77.7,"version":"GPT-OSS"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":82,"score":77.2,"version":"Phi-4"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":83,"score":76.88,"version":"GPT-OSS"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":84,"score":76.8,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":85,"score":76.7,"version":"Qwen3"},{"company":"antgroup","family":"Ring","model":"ring-lite","place":86,"score":76.6,"version":"Ring Lite"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":87,"score":76,"version":"Claude 4"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":88,"score":76,"version":"EXAONE 4.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":89,"score":76,"version":"Claude 4"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":90,"score":76,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":91,"score":76,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":92,"score":75.8,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-z1","place":93,"score":75.6,"version":"GLM Z1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":94,"score":75.6,"version":"openPangu-R"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t-chimera","place":95,"score":74.7,"version":"DeepSeek-TNG R1T"},{"company":"microsoft","family":"Phi","model":"phi-4-reasoning","place":96,"score":74.6,"version":"Phi 4"},{"company":"openai","family":"o1","model":"o1","place":97,"score":74.3,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":98,"score":74.3,"version":"o1"},{"company":"magistral","family":"Magistral","model":"magistral-medium","place":99,"score":73.6,"version":"Magistral"},{"company":"mistral","family":"Magistral","model":"magistral-medium-2506","place":100,"score":73.6,"version":"Magistral 2506"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.0","place":101,"score":73.59,"version":"Magistral 1.0"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":102,"score":73.5,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":103,"score":73.33,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":104,"score":73.3,"version":"Apriel Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":105,"score":73.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":106,"score":73.3,"version":"GPT-OSS"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":107,"score":73.3,"version":"TeleChat3"},{"company":"deepseek","family":"DeepSeek-R1","model":"deepseek-r1-distill-32b","place":108,"score":72.6,"version":"32B"},{"company":"nvidia","family":"AceReason","model":"acereason-nemotron-1.1","place":109,"score":72.6,"version":"1.1"},{"company":"amazon","family":"Nemotron","model":"questa-nemotron-1.5b","place":110,"score":72.5,"version":"QuestA Nemotron"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":111,"score":72.3,"version":"Motif 2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":112,"score":72.19,"version":"LongCat-Flash-Lite"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.1","place":113,"score":72.03,"version":"Magistral 1.1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":114,"score":71.67,"version":"Nemotron Nano V2"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":115,"score":71.1,"version":"TeleChat3"},{"company":"mistral","family":"Magistral","model":"magistral-small-2506","place":116,"score":70.7,"version":"Magistral 2506"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.0","place":117,"score":70.68,"version":"Magistral 1.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":118,"score":70.63,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.1","place":119,"score":70.52,"version":"Magistral 1.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":120,"score":70.52,"version":"Kimi Linear"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":121,"score":70.3,"version":"DeepSeek R1"},{"company":"KunLun","family":"Skywork OR1","model":"skywork-or1-7b","place":122,"score":70.2,"version":"Skywork OR1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-70b","place":123,"score":70,"version":"DeepSeek R1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":124,"score":69.6,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":125,"score":69.6,"version":"Kimi K2"},{"company":"xiaomi","family":"MiMo","model":"mimo-7b-rl","place":126,"score":68.2,"version":"MiMo 7B"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":127,"score":67.5,"version":"Llama Nemotron v1"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":128,"score":65.4,"version":"Youtu-LLM"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":129,"score":64.6,"version":"Nemotron-H"},{"company":"openai","family":"o1","model":"o1-mini","place":130,"score":63.6,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":131,"score":63.33,"version":"Gemini 2.5"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":132,"score":62,"version":"Llama 3.1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-1.5b","place":133,"score":61.77,"version":"Nemotron"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":134,"score":61.3,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":135,"score":59.4,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":136,"score":59.4,"version":"DeepSeek V3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":137,"score":55.5,"version":"Nemotron OpenReasoning"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":138,"score":54.8,"version":"ERNIE 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":139,"score":52.5,"version":"DeepSeek R1"},{"company":"xai","family":"Grok","model":"grok-3","place":140,"score":52.2,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":141,"score":48.3,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":142,"score":48.2,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":143,"score":48.1,"version":"GPT-4.1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":144,"score":46.7,"version":"SmollM3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":145,"score":46.5,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":146,"score":44.2,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":147,"score":43.4,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":148,"score":40.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":149,"score":36.7,"version":"GPT-4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":150,"score":32.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":151,"score":31.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":152,"score":31,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":153,"score":30.2,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":154,"score":29.1,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":155,"score":28.9,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":156,"score":28.33,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":157,"score":27.92,"version":"Llama 4"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":158,"score":27.5,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":159,"score":26.8,"version":"Mistral 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":160,"score":23.75,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":161,"score":22.5,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":162,"score":22.29,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-350m-math","place":163,"score":21.3,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":164,"score":18.9,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":165,"score":17.92,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":166,"score":17.29,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":167,"score":16.04,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":168,"score":15.7,"version":"Qwen2.5"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":169,"score":15.5,"version":"MobileLLM R1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":170,"score":14.37,"version":"Falcon H1"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":171,"score":13.1,"version":"MobileLLM R1"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":172,"score":12.92,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":173,"score":12.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":174,"score":12.29,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":175,"score":11.88,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":176,"score":11.67,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-6b","place":177,"score":11.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":178,"score":11.3,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":179,"score":11.25,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":180,"score":9.79,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":181,"score":8.75,"version":"Falcon 3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":182,"score":8.12,"version":"Granite 3.3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":183,"score":6.67,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":184,"score":6.25,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":185,"score":5.42,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":186,"score":4.7,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":187,"score":4.6,"version":"OLMo 2"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":188,"score":3.96,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":189,"score":3.75,"version":"Falcon H1"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":190,"score":3.28,"version":"Granite 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":191,"score":2.71,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":192,"score":2.43,"version":"Granite 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":193,"score":2.29,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":194,"score":1.97,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":195,"score":1.46,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":196,"score":1.1,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":197,"score":0.9,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":198,"score":0.89,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":199,"score":0.89,"version":"Granite 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":200,"score":0.62,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-0425-1b-instruct","place":201,"score":0.6,"version":"OLMo 2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":202,"score":0.41,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":203,"score":0.2,"version":"SmolLM2"}],"stars":"","top_company":"openai","top_model":"GPT-OSS 120B","top_model_family":"GPT","top_model_id":"gpt-oss-120b","top_model_version":"GPT-OSS","top_open_source":1,"top_percent":96.6,"topic":"Math (competition)","total_models":1613,"url":"https://artofproblemsolving.com/wiki/index.php/2024_AIME_I_Problems"},{"bench_models":4,"description":"Korean translation of AIME 2024 problems.","github":"","has_recent":0,"id":"aime-2024-ko","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AIME 2024-Ko","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":1,"score":80.3,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":2,"score":75,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":3,"score":72.3,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":4,"score":25.3,"version":"Kanana 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":80.3,"topic":"Math (competition, Korean)","total_models":1613,"url":""},{"bench_models":343,"description":"American Invitational Mathematics Examination 2025 problems.","github":"","has_recent":0,"id":"aime-2025","leaderboard":"https://www.kaggle.com/benchmarks/open-benchmarks/aime-2025","lower_is_better":false,"metric":"percent","name":"AIME 2025","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":1,"score":100,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":2,"score":100,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":3,"score":100,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":4,"score":100,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":5,"score":100,"version":"GPT-5.2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":6,"score":100,"version":"LongCat-Flash"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":7,"score":100,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":100,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":99.8,"version":"Gemini 3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":10,"score":99.6,"version":"LongCat-Flash"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":11,"score":99.2,"version":"Nemotron 3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":12,"score":99.2,"version":"Nemotron 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":13,"score":99.1,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":14,"score":99,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":15,"score":98.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":16,"score":98.7,"version":"GPT-OSS"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":17,"score":98.3,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":18,"score":98,"version":"GPT-5.2"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":19,"score":97.3,"version":"Step 3.5 Flash"},{"company":"openai","family":"GPT","model":"gpt-5-pro","place":20,"score":96.7,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":21,"score":96.3,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":22,"score":96.1,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":23,"score":96.1,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":24,"score":96,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":25,"score":96,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":26,"score":95.7,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":27,"score":95.3,"version":"GLM 4.7"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":28,"score":95.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":29,"score":95,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":30,"score":94.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":31,"score":94.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":32,"score":94.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking-high","place":33,"score":94.6,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":34,"score":94.5,"version":"Kimi K2"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":35,"score":94.43,"version":"Step3-VL"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":36,"score":94.3,"version":"Seed 1.8"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":37,"score":94.1,"version":"MiMo V2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":38,"score":94.1,"version":"MiMo V2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":39,"score":94,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.1-thinking","place":40,"score":94,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":41,"score":94,"version":"GPT-5.1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":42,"score":93.9,"version":"GLM 4.6"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":43,"score":93.5,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":44,"score":93.4,"version":"GPT-OSS"},{"company":"antgroup","family":"Ring","model":"ring-1t","place":45,"score":93.4,"version":"Ring"},{"company":"xai","family":"Grok","model":"grok-4","place":46,"score":93.3,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-0709","place":47,"score":93.3,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":48,"score":93.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":49,"score":93.1,"version":"DeepSeek V3.2"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":50,"score":93.1,"version":"Intern-S1"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":51,"score":93,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":52,"score":92.8,"version":"Claude 4.5"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":53,"score":92.8,"version":"K-EXAONE"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":54,"score":92.8,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":55,"score":92.7,"version":"Grok 4"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":56,"score":92.6,"version":"Ring"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":57,"score":92.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":58,"score":92.5,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":59,"score":92.3,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":60,"score":92,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":61,"score":92,"version":"DeepSeek V3.2"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":62,"score":91.9,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":63,"score":91.7,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-a4b","place":64,"score":91.7,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":65,"score":91.7,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":66,"score":91.7,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":67,"score":91.6,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":68,"score":91.3,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":69,"score":91.2,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":70,"score":91,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":71,"score":91,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":72,"score":90.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-2025-08-07","place":73,"score":90.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":74,"score":90.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":75,"score":90,"version":"Qwen3 VL"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":76,"score":89.7,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":77,"score":89.7,"version":"DeepSeek V3.1"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":78,"score":89.47,"version":"Cogito 671B v2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":79,"score":89.3,"version":"DeepSeek V3.2"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":80,"score":89.1,"version":"Nemotron 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":81,"score":89.06,"version":"DeepSeek V3.1"},{"company":"openai","family":"o3","model":"o3","place":82,"score":88.9,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":83,"score":88.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":84,"score":88.4,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":85,"score":88,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":86,"score":88,"version":"Claude 4.5"},{"company":"primeintellect","family":"INTELLECT","model":"intellect-3","place":87,"score":88,"version":"INTELLECT"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":88,"score":88,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":89,"score":87.8,"version":"Qwen3 Next"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":90,"score":87.7,"version":"Gemini 2.5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":91,"score":87.66,"version":"Step3-VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0528","place":92,"score":87.5,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":93,"score":87.5,"version":"DeepSeek R1"},{"company":"openai","family":"o3","model":"o3","place":94,"score":87.5,"version":"o3"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":95,"score":87.5,"version":"o3"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":96,"score":87.5,"version":"Apriel 1.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":97,"score":87,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":98,"score":87,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":99,"score":87,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":100,"score":87,"version":"Seed 2.0"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":101,"score":86.98,"version":"Ring 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":102,"score":86.7,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3-high","place":103,"score":86.58,"version":"o3"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":104,"score":85.83,"version":"dots.vlm1"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":105,"score":85.6,"version":"Nanbeige4"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":106,"score":85.3,"version":"EXAONE 4.0"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":107,"score":85.1,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":108,"score":85,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":109,"score":85,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":110,"score":84.7,"version":"Seed OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":111,"score":84.4,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":112,"score":84.3,"version":"Qwen3 Next"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":113,"score":84.3,"version":"Solar Open"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":114,"score":84.2,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":115,"score":84,"version":"Nemotron OpenReasoning"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":116,"score":83.96,"version":"Gemini 2.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":117,"score":83.67,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":118,"score":83.59,"version":"Qwen3 VL"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":119,"score":83.48,"version":"Magistral 1.2"},{"company":"zai","family":"GLM","model":"glm-4.5","place":120,"score":83.3,"version":"GLM 4.5"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":121,"score":83.3,"version":"Nemotron Cascade"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":122,"score":83.3,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":123,"score":83.1,"version":"Qwen3 VL"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":124,"score":83.1,"version":"Falcon H1R"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":125,"score":83,"version":"MiniMax M2.1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":126,"score":82.71,"version":"Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":127,"score":82.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":128,"score":82.7,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":129,"score":82.6,"version":"Qwen3"},{"company":"openai","family":"o4","model":"o4-mini","place":130,"score":82.5,"version":"o4"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":131,"score":82.5,"version":"o4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":132,"score":82.3,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":133,"score":82,"version":"Nemotron OpenReasoning"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":134,"score":82,"version":"Magistral 1.2"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":135,"score":82,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":136,"score":81.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":137,"score":81.5,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":138,"score":81.4,"version":"Nemotron Cascade"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":139,"score":81.3,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":140,"score":81.3,"version":"Qwen3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":141,"score":81.3,"version":"openPangu-R"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":142,"score":81.04,"version":"Falcon H1R"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":143,"score":80.7,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":144,"score":80.6,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":145,"score":80.3,"version":"Claude 4.1"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":146,"score":80.3,"version":"Magistral 1.2"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":147,"score":80.2,"version":"K2 V2"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":148,"score":80.1,"version":"Nemotron Cascade"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":149,"score":80,"version":"GPT-OSS"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":150,"score":80,"version":"EXAONE 4.0"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":151,"score":80,"version":"Motif 2"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":152,"score":80,"version":"Apriel 1.5"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":153,"score":79.2,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":154,"score":78.8,"version":"Qwen3 VL"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":155,"score":78.33,"version":"MiniCPM SALA"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":156,"score":78.3,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":157,"score":78.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-2025-08-05","place":158,"score":78.3,"version":"GPT-OSS"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":159,"score":78.3,"version":"MiniMax M2"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":160,"score":78.2,"version":"Nemotron OpenReasoning"},{"company":"allenai","family":"OLMo","model":"olmo-3.1-think-32b","place":161,"score":78.1,"version":"OLMo 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":162,"score":78,"version":"Claude 4.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":163,"score":78,"version":"MiniMax M2"},{"company":"antgroup","family":"Ring","model":"ring-mini-sparse-2.0-exp","place":164,"score":77.92,"version":"Ring 2.0"},{"company":"mbzuai","family":"K2","model":"k2-think","place":165,"score":77.72,"version":"K2-THINK"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":166,"score":77.7,"version":"GPT-OSS"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":167,"score":77.5,"version":"o3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":168,"score":77.34,"version":"Magistral 1.2"},{"company":"xai","family":"Grok","model":"grok-3-beta-think","place":169,"score":77.3,"version":"Grok 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":170,"score":76.9,"version":"MiniMax M1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":171,"score":76.8,"version":"DeepSeek R1"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":172,"score":76.7,"version":"Llama Nemotron v1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":173,"score":75.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-05-20","place":174,"score":75.8,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":175,"score":75.8,"version":"MiniMax M2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":176,"score":75.8,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":177,"score":75.5,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":178,"score":75.47,"version":"Qwen3"},{"company":"aquif","family":"Aquif","model":"aquif-3.6-1b","place":179,"score":75,"version":"Aquif 3.6"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":180,"score":75,"version":"GPT-OSS"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":181,"score":75,"version":"Ministral 3"},{"company":"openai","family":"o3","model":"o3-mini","place":182,"score":74.8,"version":"o3"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":183,"score":74.8,"version":"o3"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":184,"score":74.6,"version":"MiniMax M1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1","place":185,"score":74.6,"version":"MiniMax M1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":186,"score":74.58,"version":"GPT-OSS"},{"company":"weibo","family":"VibeThinker","model":"vibethinker-1.5b","place":187,"score":74.4,"version":"VibeThinker"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":188,"score":74.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":189,"score":74.3,"version":"Claude 4"},{"company":"antgroup","family":"Ring","model":"ring-mini-2.0","place":190,"score":74.06,"version":"Ring 2.0"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":191,"score":74,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":192,"score":74,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":193,"score":74,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":194,"score":73.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":195,"score":73.7,"version":"GLM 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":196,"score":73.4,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":197,"score":73.33,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":198,"score":73.3,"version":"Gemini 2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":199,"score":73.3,"version":"OLMo 3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":200,"score":73.3,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":201,"score":72.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":202,"score":72.9,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":203,"score":72.7,"version":"Kanana 2"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-ultra-253b-v1","place":204,"score":72.5,"version":"Llama 3.1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":205,"score":72.5,"version":"OLMo 3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":206,"score":72.1,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":207,"score":72.1,"version":"Nemotron Nano v2"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":208,"score":72.08,"version":"MiniCPM 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":209,"score":72,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":210,"score":72,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":211,"score":72,"version":"Nemotron Nano v2"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":212,"score":71.88,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":213,"score":71.25,"version":"Qwen3"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":214,"score":71.2,"version":"Phi-4"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":215,"score":71,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":216,"score":70.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":217,"score":70.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":218,"score":70.9,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":219,"score":70.7,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":220,"score":70.7,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":221,"score":70.42,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":222,"score":70.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":223,"score":70.4,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":224,"score":70.1,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":225,"score":70,"version":"DeepSeek R1"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t2-chimera","place":226,"score":70,"version":"DeepSeek-TNG R1T2"},{"company":"openai","family":"o1","model":"o1","place":227,"score":70,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":228,"score":70,"version":"o1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0120","place":229,"score":70,"version":"0120"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":230,"score":69.7,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":231,"score":69.5,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":232,"score":69.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":233,"score":69.3,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":234,"score":69.2,"version":"Claude 4"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":235,"score":69.17,"version":"QwQ"},{"company":"antgroup","family":"Ring","model":"ring-lite","place":236,"score":69.1,"version":"Ring Lite"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":237,"score":68.7,"version":"QwQ"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":238,"score":68.44,"version":"Qwen3 Next"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":239,"score":68.3,"version":"OLMo 3"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":240,"score":67.5,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":241,"score":67.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":242,"score":66.67,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":243,"score":66.2,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":244,"score":65.9,"version":"Qwen3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":245,"score":65.83,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":246,"score":65.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking","place":247,"score":65.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":248,"score":65,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.0","place":249,"score":64.95,"version":"Magistral 1.0"},{"company":"magistral","family":"Magistral","model":"magistral-medium","place":250,"score":64.9,"version":"Magistral"},{"company":"mistral","family":"Magistral","model":"magistral-medium-2506","place":251,"score":64.9,"version":"Magistral 2506"},{"company":"nvidia","family":"AceReason","model":"acereason-nemotron-1.1","place":252,"score":64.8,"version":"1.1"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":253,"score":64.7,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":254,"score":64.2,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":255,"score":64.06,"version":"Seed 1.5"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":256,"score":63.6,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl-thinking","place":257,"score":63.5,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":258,"score":63.33,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":259,"score":63.33,"version":"LLaDA 2.1"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":260,"score":63.23,"version":"LongCat-Flash-Lite"},{"company":"microsoft","family":"Phi","model":"phi-4-reasoning","place":261,"score":63.1,"version":"Phi 4"},{"company":"mistral","family":"Magistral","model":"magistral-small-2506","place":262,"score":62.8,"version":"Magistral 2506"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.0","place":263,"score":62.76,"version":"Magistral 1.0"},{"company":"amazon","family":"Nemotron","model":"questa-nemotron-1.5b","place":264,"score":62.29,"version":"QuestA Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":265,"score":62.08,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.1","place":266,"score":62.03,"version":"Magistral 1.1"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":267,"score":62,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":268,"score":61.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":269,"score":61.88,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":270,"score":61.7,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":271,"score":61.3,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.1","place":272,"score":60.99,"version":"Magistral 1.1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":273,"score":60.6,"version":"openPangu-R"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":274,"score":60,"version":"Apriel Nemotron"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":275,"score":60,"version":"Llama Nemotron v1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":276,"score":60,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":277,"score":60,"version":"LLaDA 2.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":278,"score":59.58,"version":"Kimi Linear"},{"company":"openai","family":"GPT","model":"gpt-5","place":279,"score":59.43,"version":"GPT-5"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":280,"score":59,"version":"HyperCLOVA X"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":281,"score":58.6,"version":"Kimi Linear"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t-chimera","place":282,"score":58.3,"version":"DeepSeek-TNG R1T"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":283,"score":57.3,"version":"Kimi K2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":284,"score":56.67,"version":"Nemotron Nano V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":285,"score":56.3,"version":"DeepSeek R1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":286,"score":55.9,"version":"Ling 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":287,"score":55.89,"version":"Ling 2.0"},{"company":"xiaomi","family":"MiMo","model":"mimo-7b-rl","place":288,"score":55.4,"version":"MiMo 7B"},{"company":"zai","family":"GLM","model":"glm-z1","place":289,"score":55.4,"version":"GLM Z1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":290,"score":55.21,"version":"DeepSeek V3.1"},{"company":"openai","family":"o1","model":"o1-mini","place":291,"score":54.8,"version":"o1"},{"company":"KunLun","family":"Skywork OR1","model":"skywork-or1-7b","place":292,"score":54.6,"version":"Skywork OR1"},{"company":"xai","family":"Grok","model":"grok-3","place":293,"score":53.3,"version":"Grok 3"},{"company":"deepseek","family":"DeepSeek-R1","model":"deepseek-r1-distill-32b","place":294,"score":51.8,"version":"32B"},{"company":"openai","family":"o1","model":"o1-mini-2024-09-12","place":295,"score":51.7,"version":"o1"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":296,"score":51.4,"version":"Nemotron-H"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":297,"score":51,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":298,"score":50.16,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":299,"score":50.1,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":300,"score":49.8,"version":"DeepSeek V3.1"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":301,"score":49.8,"version":"Youtu-LLM"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":302,"score":49.5,"version":"Kimi K2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-1.5b","place":303,"score":49.5,"version":"Nemotron"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":304,"score":49.5,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":305,"score":49.2,"version":"Kimi K2"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":306,"score":48.7,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":307,"score":48.1,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":308,"score":47.9,"version":"EXAONE Deep"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":309,"score":47.66,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":310,"score":47.4,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":311,"score":47,"version":"Ling 2.0"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":312,"score":46.9,"version":"K2 V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":313,"score":46.7,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":314,"score":46.7,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":315,"score":46.6,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":316,"score":46.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":317,"score":45.9,"version":"Qwen3 VL"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":318,"score":45.6,"version":"Nemotron OpenReasoning"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":319,"score":45.2,"version":"EXAONE 4.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":320,"score":43.33,"version":"LLaDA 2.1"},{"company":"essential","family":"Rnj","model":"rnj-1","place":321,"score":43.3,"version":"Rnj 1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":322,"score":37.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":323,"score":37,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":324,"score":36.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":325,"score":36.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":326,"score":36.7,"version":"GPT-4.1"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":327,"score":36.7,"version":"SmolLM3"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":328,"score":36.7,"version":"GPT-4.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":329,"score":36.67,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":330,"score":36.67,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":331,"score":36.27,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":332,"score":35.8,"version":"Claude 4"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":333,"score":35.1,"version":"ERNIE 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":334,"score":34.4,"version":"DeepSeek R1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":335,"score":34.2,"version":"SmollM3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":336,"score":33.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":337,"score":33.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1-20250805","place":338,"score":33.3,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":339,"score":33.1,"version":"Claude 4"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":340,"score":31.73,"version":"LFM2.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":341,"score":31.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":342,"score":29.2,"version":"Gemini 2.0"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-nano-1b","place":343,"score":28.7,"version":"Aquif 3.5"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":344,"score":27.3,"version":"K2 V2"},{"company":"liquidai","family":"LFM","model":"lfm2-350m-math","place":345,"score":27.1,"version":"LFM2"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":346,"score":26.7,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":347,"score":26.4,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":348,"score":25.8,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":349,"score":25,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":350,"score":24.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":351,"score":24.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":352,"score":24.17,"version":"GLM 4.7"},{"company":"arcee","family":"Trinity","model":"trinity-large-preview","place":353,"score":24,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-nothinking-2504","place":354,"score":23.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":355,"score":23.3,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":356,"score":23.3,"version":"LLaDA 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":357,"score":23.1,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":358,"score":22.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":359,"score":22.71,"version":"Gemma 3"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b-exp","place":360,"score":22.67,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":361,"score":22.08,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":362,"score":21.6,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":363,"score":21.2,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":364,"score":20.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":365,"score":20.9,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":366,"score":20.8,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":367,"score":20.2,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-12b-it","place":368,"score":20,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":369,"score":19.3,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":370,"score":19.17,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":371,"score":18.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":372,"score":18.75,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl","place":373,"score":18.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-6b","place":374,"score":17.8,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":375,"score":16.67,"version":"Falcon H1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-21b-a3b-pt","place":376,"score":16.1,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":377,"score":15.21,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":378,"score":15.2,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":379,"score":15.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":380,"score":15,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4","place":381,"score":15,"version":"GPT-4"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":382,"score":14.7,"version":"OLMo 3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":383,"score":14,"version":"LFM2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":384,"score":13.96,"version":"Falcon H1"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-2.7b","place":385,"score":13.4,"version":"Aquif 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":386,"score":13.4,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":387,"score":13.33,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":388,"score":12.7,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":389,"score":12.5,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-2025-08-05","place":390,"score":11.7,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":391,"score":11.46,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":392,"score":11.04,"version":"Falcon H1"},{"company":"xai","family":"Grok","model":"grok-2-1212","place":393,"score":10.8,"version":"Grok 2"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":394,"score":10.8,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":395,"score":10,"version":"Gemini 1.5"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":396,"score":10,"version":"SmollM3"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":397,"score":9.83,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":398,"score":9.8,"version":"Qwen3"},{"company":"aquif","family":"Aquif","model":"aquif-3-3.2b","place":399,"score":9.6,"version":"Aquif 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":400,"score":9.58,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":401,"score":9.58,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":402,"score":9.33,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":403,"score":9.3,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":404,"score":8.96,"version":"Llama 4"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-it","place":405,"score":8.67,"version":"Gemma 3n"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":406,"score":8.4,"version":"MobileLLM R1"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":407,"score":8.3,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":408,"score":8,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-001","place":409,"score":7.5,"version":"Gemini 1.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":410,"score":7.2,"version":"OLMo 3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":411,"score":6.25,"version":"Falcon 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":412,"score":5.8,"version":"GPT-4o"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":413,"score":5.42,"version":"Falcon 3"},{"company":"microsoft","family":"Phi","model":"phi-4-mini-3.8b","place":414,"score":5.3,"version":"Phi 4 mini"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b","place":415,"score":4.5,"version":"EXAONE 3.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":416,"score":4.38,"version":"Falcon H1"},{"company":"servicenow","family":"Apriel","model":"apriel-5b-instruct","place":417,"score":4.3,"version":"Apriel"},{"company":"mistral","family":"Mistral","model":"mistral-large-2411","place":418,"score":4.2,"version":"Mistral Large"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":419,"score":3.33,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":420,"score":2.9,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":421,"score":2.7,"version":"Llama 3.1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":422,"score":2.5,"version":"Claude 3.5"},{"company":"mistral","family":"Codestral","model":"codestral-2501","place":423,"score":2.5,"version":"Codestral"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":424,"score":2.29,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":425,"score":1.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":426,"score":1.7,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":427,"score":1.67,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":428,"score":1.25,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":429,"score":1.25,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":430,"score":1.25,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":431,"score":1,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":432,"score":1,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":433,"score":1,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":434,"score":0.9,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":435,"score":0.9,"version":"OLMo 2"},{"company":"mistral","family":"Ministral","model":"ministral-3b-2410","place":436,"score":0.8,"version":"Ministral"},{"company":"mistral","family":"Ministral","model":"ministral-8b-2410","place":437,"score":0.8,"version":"Ministral"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":438,"score":0.67,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":439,"score":0.42,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":440,"score":0.33,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":441,"score":0.33,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":442,"score":0.3,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":443,"score":0.21,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":444,"score":0.21,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":445,"score":0.21,"version":"Qwen2.5"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":446,"score":0.2,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":447,"score":0.1,"version":"Llama 3.0"},{"company":"swissai","family":"Apertus","model":"apertus-70b-instruct","place":448,"score":0.1,"version":"Apertus"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":449,"score":0,"version":"Llama 3.2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-1106","place":450,"score":0,"version":"GPT-3.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku-20241022","place":451,"score":0,"version":"Claude 3.5"},{"company":"mistral","family":"Mixtral","model":"open-mixtral-8x22b-2404","place":452,"score":0,"version":"Mixtral"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":453,"score":0,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":454,"score":0,"version":"Qwen2.5"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":455,"score":0,"version":"Codestral"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.5","top_model_family":"Claude","top_model_id":"claude-4.5-sonnet","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":100,"topic":"Math (competition)","total_models":1613,"url":"https://artofproblemsolving.com/wiki/index.php/AIME_Problems_and_Solutions"},{"bench_models":27,"description":"American Invitational Mathematics Examination 2026 I problems.","github":"","has_recent":0,"id":"aime-2026-i","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AIME 2026 I","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":97.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":96.7,"version":"GPT-5.2"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5-heavy-thinking","place":3,"score":95,"version":"Ring 2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":4,"score":94.2,"version":"Seed 2.0"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5","place":5,"score":93.75,"version":"Ring 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":93.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":7,"score":93.3,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":93.3,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":9,"score":93.3,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":10,"score":93.3,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":11,"score":92.71,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":12,"score":92.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":13,"score":92.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":14,"score":91.3,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":15,"score":90.62,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":90.6,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":17,"score":88.3,"version":"Seed 2.0"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":18,"score":87.4,"version":"Nanbeige4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":19,"score":87.3,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":20,"score":87.08,"version":"Ling 2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":21,"score":86.7,"version":"Seed 2.0"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":22,"score":84.1,"version":"Nanbeige4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":23,"score":82.5,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":24,"score":81.46,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":25,"score":76.46,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":26,"score":75.83,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":27,"score":75.16,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":28,"score":70.42,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":29,"score":66.98,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":30,"score":66.41,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":31,"score":66.2,"version":"GPT-5.2"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":97.5,"topic":"Math (competition)","total_models":1613,"url":""},{"bench_models":5,"description":"AInstein agent coding benchmark.","github":"","has_recent":0,"id":"ainstein-swe-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AInstein-SWE-Bench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":42.8,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":36.7,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":35.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":33.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":19.3,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":42.8,"topic":"Agentic coding","total_models":1613,"url":""},{"bench_models":3,"description":"Benchmark for instruction-following quality and alignment behavior.","github":"","has_recent":0,"id":"alignbench","leaderboard":"","lower_is_better":false,"metric":"score","name":"AlignBench","paper":"","placements":[{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":1,"score":8.24,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":2,"score":8.07,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":3,"score":6.85,"version":"GLM 4.7"}],"stars":"","top_company":"jd","top_model":"JoyAI-LLM Flash","top_model_family":"JoyAI-LLM","top_model_id":"joyai-llm-flash","top_model_version":"JoyAI-LLM","top_open_source":1,"top_percent":8.24,"topic":"Alignment and instruction following","total_models":1613,"url":""},{"bench_models":10,"description":"All-Angles benchmark for spatial recognition and 3D perception.","github":"","has_recent":0,"id":"all-angles-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"All-Angles Bench","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":1,"score":57.21,"version":"Step3-VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":2,"score":56.9,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":3,"score":53.24,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":4,"score":52.7,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":5,"score":52.4,"version":"Step 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":6,"score":51.62,"version":"MiMo VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":7,"score":48.9,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":8,"score":48.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":9,"score":45.88,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":10,"score":45.29,"version":"InternVL 3.5"}],"stars":"","top_company":"stepfun","top_model":"Step3-VL-10B","top_model_family":"Step","top_model_id":"step3-vl-10b","top_model_version":"Step3-VL","top_open_source":1,"top_percent":57.21,"topic":"Spatial perception","total_models":1613,"url":""},{"bench_models":80,"description":"Automatic eval using GPT-4 as a judge.","github":"","has_recent":0,"id":"alpacaeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AlpacaEval","paper":"","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":99.4,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":2,"score":99.3,"version":"Llama 3.3"},{"company":"openai","family":"GPT","model":"gpt-4-1106-preview","place":3,"score":97.7,"version":"GPT-4"},{"company":"mistral","family":"Mistral","model":"mistral-medium","place":4,"score":96.8,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":5,"score":96.1,"version":"Llama 3.3"},{"company":"openai","family":"GPT","model":"gpt-4","place":6,"score":95.3,"version":"GPT-4"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b-instruct-v0.1","place":7,"score":94.8,"version":"Mixtral v0.1"},{"company":"openai","family":"GPT","model":"gpt-4-0314","place":8,"score":94.8,"version":"GPT-4"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":9,"score":94.5,"version":"Llama TFree HAT"},{"company":"01ai","family":"Yi","model":"yi-34b-chat","place":10,"score":94.1,"version":"Yi 34B Chat"},{"company":"openai","family":"GPT","model":"gpt-4-0613","place":11,"score":93.8,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0613","place":12,"score":93.4,"version":"GPT-3.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":13,"score":92.8,"version":"Mistral v0.2"},{"company":"meta","family":"Llama","model":"llama-2-70b-chat","place":14,"score":92.7,"version":"Llama 2"},{"company":"anthropic","family":"Claude","model":"claude","place":15,"score":91.6,"version":"Claude 1"},{"company":"anthropic","family":"claude-2","model":"claude-2","place":16,"score":91.4,"version":"claude-2 (Mar 2024)"},{"company":"cohere","family":"Command","model":"command","place":17,"score":90.6,"version":"Command"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":18,"score":89.4,"version":"GPT-3.5"},{"company":"lmsys","family":"Vicuna","model":"vicuna-33b-v1.3","place":19,"score":89,"version":"Vicuna"},{"company":"anthropic","family":"Claude","model":"claude-2.1","place":20,"score":87.1,"version":"Claude 2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-1106","place":21,"score":86.3,"version":"GPT-3.5"},{"company":"microsoft","family":"Phi","model":"phi-2-dpo","place":22,"score":82.3,"version":"Phi"},{"company":"lmsys","family":"Vicuna","model":"vicuna-13b-v1.3","place":23,"score":82.1,"version":"Vicuna"},{"company":"meta","family":"Llama","model":"llama-2-13b-chat","place":24,"score":81.1,"version":"Llama 2"},{"company":"google","family":"Gemini","model":"gemini-pro","place":25,"score":79.7,"version":"Gemini"},{"company":"lmsys","family":"Vicuna","model":"vicuna-7b-v1.3","place":26,"score":76.8,"version":"Vicuna"},{"company":"wizardlm","family":"WizardLM","model":"wizard-13b","place":27,"score":75.3,"version":"WizardLM"},{"company":"timdettmers","family":"Guanaco","model":"guanaco-65b","place":28,"score":71.8,"version":"Guanaco"},{"company":"meta","family":"Llama","model":"llama-2-7b-chat","place":29,"score":71.4,"version":"Llama 2"},{"company":"lmsys","family":"Vicuna-13B","model":"vicuna-13b","place":30,"score":70.4,"version":"Vicuna-13B"},{"company":"microsoft","family":"Phi","model":"phi-2-sft","place":31,"score":68.5,"version":"Phi"},{"company":"openassistant","family":"LLaMA","model":"llama-33b-oasst-rlhf","place":32,"score":66.5,"version":"LLaMA OASST"},{"company":"timdettmers","family":"Guanaco","model":"guanaco-33b","place":33,"score":66,"version":"Guanaco-33B"},{"company":"nousresearch","family":"Nous Hermes","model":"nous-hermes-13b","place":34,"score":65.5,"version":"Nous Hermes"},{"company":"lmsys","family":"Vicuna-7B","model":"vicuna-7b","place":35,"score":64.4,"version":"Vicuna-7B"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":36,"score":64.21,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":37,"score":56.16,"version":"Gemma 3"},{"company":"openassistant","family":"LLaMA","model":"llama-33b-oasst-sft","place":38,"score":55,"version":"LLaMA OASST"},{"company":"timdettmers","family":"Guanaco","model":"guanaco-13b","place":39,"score":52.6,"version":"Guanaco"},{"company":"openai","family":"Davinci","model":"davinci-003","place":40,"score":50,"version":"Davinci"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":41,"score":49.29,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":42,"score":48.32,"version":"Falcon H1"},{"company":"timdettmers","family":"Guanaco","model":"guanaco-7b","place":43,"score":46.6,"version":"Guanaco"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":44,"score":46.13,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-40b-instruct","place":45,"score":45.7,"version":"Falcon"},{"company":"tii","family":"Falcon","model":"falcon-7b-instruct","place":46,"score":45.7,"version":"Falcon"},{"company":"stanford","family":"Alpaca Farm","model":"alpaca-farm-ppo-sim-gpt4-7b","place":47,"score":44.1,"version":"Alpaca Farm"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":48,"score":43.55,"version":"Gemma 3"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b-sft","place":49,"score":41.9,"version":"Pythia"},{"company":"stanford","family":"Alpaca Farm","model":"alpaca-farm-ppo-human-7b","place":50,"score":41.2,"version":"Alpaca Farm"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":51,"score":40.23,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":52,"score":39.64,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":53,"score":39.26,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":54,"score":38.27,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":55,"score":36.51,"version":"Qwen3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":56,"score":36.3,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":57,"score":36.26,"version":"Llama 4"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":58,"score":31.09,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":59,"score":29.48,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":60,"score":28.18,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":61,"score":27.56,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":62,"score":27.12,"version":"Falcon H1"},{"company":"stanford","family":"Alpaca","model":"alpaca-7b","place":63,"score":26.5,"version":"Alpaca"},{"company":"openassistant","family":"Pythia","model":"pythia-12b-oasst-sft","place":64,"score":26,"version":"Pythia"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":65,"score":25.48,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":66,"score":24.31,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":67,"score":21.89,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":68,"score":19.69,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":69,"score":17.87,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":70,"score":17.37,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":71,"score":16.8,"version":"Llama 3.3"},{"company":"openai","family":"Davinci","model":"davinci-001","place":72,"score":15.2,"version":"Davinci"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":73,"score":14.82,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":74,"score":10.79,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":75,"score":9.59,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":76,"score":9.54,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":77,"score":9.38,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":78,"score":6.98,"version":"Falcon 3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":79,"score":3.76,"version":"LFM2.5-Audio"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":80,"score":3.72,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":81,"score":3.71,"version":"LFM2-Audio"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":82,"score":3.26,"version":"Qwen2.5"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":83,"score":2.32,"version":"Mini-Omni2"},{"company":"kyutai","family":"Moshi","model":"moshi","place":84,"score":2.01,"version":"Moshi"}],"stars":"1849","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":99.4,"topic":"Instruction following","total_models":1613,"url":"https://github.com/tatsu-lab/alpaca_eval"},{"bench_models":46,"description":"Updated AlpacaEval with improved prompts and judging.","github":"","has_recent":0,"id":"alpacaeval-2.0","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AlpacaEval 2.0","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":1,"score":87.6,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":2,"score":81.9,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":3,"score":80.9,"version":"Qwen3 VL"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":4,"score":78.6,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":5,"score":75.6,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":6,"score":74.2,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":7,"score":69.1,"version":"OLMo 3"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":8,"score":66.1,"version":"Llama 3.1 Nemotron"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":9,"score":65.5,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":10,"score":65,"version":"GPT-4o"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":11,"score":62.68,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":12,"score":61.19,"version":"Granite 3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":13,"score":53.5,"version":"DeepSeek V3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":14,"score":51.4,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":15,"score":49.8,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":16,"score":49.8,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":17,"score":49.6,"version":"Tulu 3"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":18,"score":49,"version":"Magpie"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":19,"score":47.7,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":20,"score":43.7,"version":"Gemma 2"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":21,"score":43.45,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":22,"score":42.48,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":23,"score":39.8,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":24,"score":38.5,"version":"Llama 3.1"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":25,"score":38,"version":"OLMo 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":26,"score":35.16,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":27,"score":34.51,"version":"Granite 3.2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":28,"score":34.5,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":29,"score":33.5,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":30,"score":33.4,"version":"Llama 3.1"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":31,"score":31.4,"version":"Ministral"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":32,"score":30.47,"version":"Granite 4.0"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":33,"score":30.4,"version":"Tulu 3"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":34,"score":30.34,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":35,"score":30.23,"version":"Granite 4.0"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":36,"score":30.2,"version":"Hermes 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":37,"score":29.49,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":38,"score":29,"version":"Qwen2.5"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":39,"score":28.4,"version":"Hermes 3"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":40,"score":27.17,"version":"Granite 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":41,"score":26.3,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":42,"score":26.2,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":43,"score":24.2,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":44,"score":21.85,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":45,"score":15.35,"version":"DeepSeek R1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":46,"score":12.4,"version":"Tulu 3"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek R1","top_model_family":"DeepSeek","top_model_id":"deepseek-r1","top_model_version":"DeepSeek R1","top_open_source":1,"top_percent":87.6,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":45,"description":"American Mathematics Competition 2023 evaluation.","github":"","has_recent":0,"id":"amc-23","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AMC-23","paper":"","placements":[{"company":"qwen","family":"QwQ","model":"qwq-32b","place":1,"score":98.5,"version":"QwQ"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":2,"score":95,"version":"Apriel Nemotron"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":3,"score":95,"version":"EXAONE 4.0"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":4,"score":93.5,"version":"Llama 3.1"},{"company":"openai","family":"o1","model":"o1-mini","place":5,"score":92.5,"version":"o1"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":6,"score":77.81,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":7,"score":70.78,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":8,"score":69.38,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":9,"score":69.06,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":10,"score":68.75,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":11,"score":67.34,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":12,"score":67.34,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-350m-math","place":13,"score":67.2,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":14,"score":66.88,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":15,"score":66.88,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":16,"score":56.72,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":17,"score":56.56,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":18,"score":55.63,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":19,"score":53.91,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-large-3-base","place":20,"score":52,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-6b","place":21,"score":51.9,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":22,"score":48.12,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":23,"score":46.09,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":24,"score":45.78,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":25,"score":43.59,"version":"Falcon H1"},{"company":"mistral","family":"Mathstral","model":"mathstral-7b","place":26,"score":42.4,"version":"Mathstral"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":27,"score":40.2,"version":"MobileLLM R1"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":28,"score":40,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":29,"score":39.84,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":30,"score":39.38,"version":"Llama 3.3"},{"company":"zai","family":"GLM","model":"glm4-9b","place":31,"score":36,"version":"GLM 4"},{"company":"qwen","family":"Qwen","model":"qwen2-7b","place":32,"score":35.2,"version":"Qwen2"},{"company":"meta","family":"Llama","model":"llama3-8b","place":33,"score":34.4,"version":"Llama 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":34,"score":33.13,"version":"Falcon H1"},{"company":"google","family":"Gemma 2","model":"gemma2-9b","place":35,"score":31.2,"version":"Gemma 2"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":36,"score":29.69,"version":"Falcon 3"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":37,"score":28,"version":"DeepSeek Math"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":38,"score":27.97,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":39,"score":24.06,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":40,"score":22.81,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":41,"score":22.66,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":42,"score":19.22,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":43,"score":12.5,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":44,"score":7.19,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":45,"score":6.87,"version":"Falcon 3"}],"stars":"","top_company":"qwen","top_model":"QwQ-32B","top_model_family":"QwQ","top_model_id":"qwq-32b","top_model_version":"QwQ","top_open_source":1,"top_percent":98.5,"topic":"Math (competition)","total_models":1613,"url":"https://www.maa.org/math-competitions/amc-1012"},{"bench_models":19,"description":"Advanced math olympiad-style benchmark.","github":"","has_recent":0,"id":"amobench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AMO-Bench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":72.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":2,"score":66,"version":"Claude 4.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":3,"score":66,"version":"LongCat-Flash"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":64,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":5,"score":62.4,"version":"GLM 4.7"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":6,"score":61.6,"version":"LongCat-Flash"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":7,"score":60,"version":"Seed 1.8"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":8,"score":56,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":9,"score":51.9,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":10,"score":50,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":11,"score":47.8,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":38.7,"version":"Gemini 2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":13,"score":36.3,"version":"Falcon H1R"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":32,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":15,"score":26,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":16,"score":23.3,"version":"DeepSeek R1"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":17,"score":22.2,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":18,"score":21.3,"version":"Qwen3"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":19,"score":15,"version":"Phi-4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":20,"score":14.1,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":21,"score":7,"version":"Nemotron-H"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":72.5,"topic":"Math (competition)","total_models":1613,"url":""},{"bench_models":7,"description":"Chinese subset of AMO-Bench.","github":"","has_recent":0,"id":"amobench-zh","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AMO-Bench CH","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":74.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":2,"score":67.7,"version":"Claude 4.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":3,"score":67.5,"version":"LongCat-Flash"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":4,"score":56.8,"version":"LongCat-Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":5,"score":52,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":6,"score":51.8,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":7,"score":35.1,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":8,"score":28.8,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":74.9,"topic":"Math (competition)","total_models":1613,"url":""},{"bench_models":21,"description":"Benchmark for agents operating Android apps via UI automation.","github":"","has_recent":0,"id":"androidworld","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AndroidWorld","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":1,"score":71.1,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":70.7,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":69.7,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":66.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":66.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":6,"score":64.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":7,"score":63.7,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":8,"score":62.1,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":9,"score":57.3,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":10,"score":57,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":11,"score":57,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":56,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":13,"score":54.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":14,"score":47.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":15,"score":45.3,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":16,"score":42.7,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":17,"score":41.7,"version":"GLM 4.1V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":18,"score":36.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":19,"score":35,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":20,"score":30.5,"version":"GPT-5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":21,"score":4.4,"version":"Gemma 3"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-35B-A3B","top_model_family":"Qwen","top_model_id":"qwen3.5-35b-a3b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":71.1,"topic":"Mobile agents","total_models":1613,"url":""},{"bench_models":18,"description":"APEX benchmark evaluating agents on long-horizon professional tasks.","github":"","has_recent":0,"id":"apex-agents","leaderboard":"","lower_is_better":false,"metric":"percent","name":"APEX-Agents","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":33.5,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":29.8,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":3,"score":27.6,"version":"GPT-5.2 Codex"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":24,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":23,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.1-codex","place":6,"score":20.6,"version":"GPT-5.1 Codex"},{"company":"openai","family":"GPT","model":"gpt-5-codex","place":7,"score":20,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":18.4,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":18.4,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":10,"score":18.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":11,"score":17.5,"version":"GPT-5.1"},{"company":"xai","family":"Grok","model":"grok-4","place":12,"score":15.2,"version":"Grok 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":13,"score":14.4,"version":"Kimi K2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":14,"score":6.2,"version":"MiniMax M2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":15,"score":4.7,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":16,"score":4,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":17,"score":3.1,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.6","place":18,"score":3,"version":"GLM 4.6"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":33.5,"topic":"Long horizon professional tasks","total_models":1613,"url":""},{"bench_models":5,"description":"API-Bank tool-use benchmark.","github":"","has_recent":0,"id":"api-bank","leaderboard":"","lower_is_better":false,"metric":"percent","name":"API-Bank","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":1,"score":92,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":2,"score":90,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":3,"score":85.1,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":4,"score":82.6,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":5,"score":48.3,"version":"Llama 3"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B","top_model_family":"Llama","top_model_id":"llama-3.1-405b","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":92,"topic":"Tool use","total_models":1613,"url":"https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank"},{"bench_models":97,"description":"ARC-AGI Phase 1 aggregate accuracy.","github":"","has_recent":0,"id":"arc-agi-1","leaderboard":"https://arcprize.org/leaderboard","lower_is_better":false,"metric":"percent","name":"ARC-AGI-1","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":89.9,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":86.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":3,"score":86.2,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":4,"score":85.4,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":85,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":84,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":80,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3","place":8,"score":75.7,"version":"o3"},{"company":"openai","family":"o3","model":"o3-preview-low","place":9,"score":75.7,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":10,"score":75.7,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":11,"score":75,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1-thinking","place":12,"score":72.8,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":70.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-pro","place":14,"score":70.2,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":15,"score":67.9,"version":"Seed 1.8"},{"company":"xai","family":"Grok","model":"grok-4","place":16,"score":66.7,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":17,"score":66.7,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":18,"score":65.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":19,"score":65.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking-high","place":20,"score":65.7,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-32k","place":21,"score":63.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":22,"score":63.7,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3-high","place":23,"score":60.8,"version":"o3"},{"company":"openai","family":"o3","model":"o3-pro","place":24,"score":59.3,"version":"o3"},{"company":"openai","family":"o3","model":"o3-pro-high","place":25,"score":59.3,"version":"o3"},{"company":"openai","family":"o4","model":"o4-mini","place":26,"score":58.7,"version":"o4"},{"company":"openai","family":"o4","model":"o4-mini-high","place":27,"score":58.7,"version":"o4"},{"company":"openai","family":"o3","model":"o3-pro-medium","place":28,"score":57,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-medium","place":29,"score":56.2,"version":"GPT-5"},{"company":"arcprize","family":"ARChitects","model":"architects","place":30,"score":56,"version":"ARChitects"},{"company":"antgroup","family":"Ring","model":"ring-1t","place":31,"score":55.94,"version":"Ring"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":32,"score":54.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":33,"score":54.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":34,"score":54.3,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-medium","place":35,"score":53.8,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5","place":36,"score":51.88,"version":"GPT-5"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":37,"score":50.8,"version":"Ring"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-16k","place":38,"score":48.3,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":39,"score":48.12,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-8k","place":40,"score":46.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":41,"score":45.44,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3-pro-low","place":42,"score":44.3,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":43,"score":44.25,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-5-low","place":44,"score":44,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":45,"score":43.81,"version":"Ling 1T"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":46,"score":43.3,"version":"Seed 2.0"},{"company":"openai","family":"o4","model":"o4-mini-medium","place":47,"score":41.8,"version":"o4"},{"company":"openai","family":"o3","model":"o3-low","place":48,"score":41.5,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":49,"score":41,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-thinking-16k","place":50,"score":41,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":51,"score":40.62,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-16k","place":52,"score":40,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-medium","place":53,"score":37.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-thinking-32k","place":54,"score":37,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":55,"score":37,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514-thinking-16k","place":56,"score":35.7,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-mini","place":57,"score":34.5,"version":"o3"},{"company":"openai","family":"o3","model":"o3-mini-high","place":58,"score":34.5,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-16k","place":59,"score":33.3,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":60,"score":33.3,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":61,"score":33.06,"version":"Ring 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":62,"score":33,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-24k","place":63,"score":32.3,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":64,"score":32,"version":"Seed OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-thinking-1k","place":65,"score":31.3,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-1k","place":66,"score":31,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":67,"score":30.88,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514-thinking-8k","place":68,"score":30.7,"version":"Claude 4"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":69,"score":30.44,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-thinking-8k","place":70,"score":29.5,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-8k","place":71,"score":29,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking-16k","place":72,"score":28.6,"version":"Claude 3.7"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":73,"score":28.31,"version":"Ring 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-1k","place":74,"score":28,"version":"Claude 4"},{"company":"openai","family":"Codex","model":"codex-mini-latest","place":75,"score":27.3,"version":"Codex"},{"company":"openai","family":"GPT","model":"gpt-5-mini-low","place":76,"score":26.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-8k","place":77,"score":25.8,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":78,"score":25.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":79,"score":23.8,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":80,"score":22.5,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":81,"score":22.3,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5","place":82,"score":22.19,"version":"GPT-5"},{"company":"openai","family":"o4","model":"o4-mini-low","place":83,"score":21.3,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":84,"score":21.2,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking-8k","place":85,"score":21.2,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":86,"score":20.7,"version":"GPT-5"},{"company":"antgroup","family":"Ring","model":"ring-mini-2.0","place":87,"score":20.12,"version":"Ring 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":88,"score":18.94,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":89,"score":18.62,"version":"GPT-OSS"},{"company":"arcprize","family":"ARChitects","model":"icecuber","place":90,"score":17,"version":"ARChitects"},{"company":"openai","family":"GPT","model":"gpt-5-nano-high","place":91,"score":16.7,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":92,"score":16.5,"version":"Grok 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-1k","place":93,"score":16,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":94,"score":15.8,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":95,"score":14.69,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":96,"score":14.06,"version":"Kimi K2"},{"company":"openai","family":"o1","model":"o1-mini","place":97,"score":14,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":98,"score":13.6,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking-1k","place":99,"score":11.6,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":100,"score":11,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":101,"score":10.3,"version":"GPT-4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":102,"score":8.75,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":103,"score":5.5,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":104,"score":5.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":105,"score":4.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":106,"score":4.38,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":89.9,"topic":"General reasoning","total_models":1613,"url":"https://arcprize.org/"},{"bench_models":104,"description":"ARC-AGI Phase 2 aggregate accuracy.","github":"","has_recent":0,"id":"arc-agi-2","leaderboard":"https://arcprize.org/leaderboard","lower_is_better":false,"metric":"percent","name":"ARC-AGI-2","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-deep-think","place":1,"score":84.6,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":2,"score":77.1,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":3,"score":68.8,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":4,"score":68.8,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":5,"score":58.3,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":6,"score":57.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":54.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":54.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":9,"score":52.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":10,"score":52.9,"version":"GPT-5.2"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":11,"score":47.25,"version":"Ling 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":45.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":45.1,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":14,"score":43.3,"version":"GPT-5.2"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":15,"score":43.19,"version":"Ling 1T"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":16,"score":37.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":17,"score":37.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":18,"score":37.5,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":19,"score":34.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":20,"score":33.6,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":21,"score":31.19,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":22,"score":31.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":23,"score":30.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":24,"score":29.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":25,"score":24.19,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":26,"score":20.06,"version":"DeepSeek V3.2"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5","place":27,"score":19.31,"version":"Ring 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-pro","place":28,"score":18.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":29,"score":17.6,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.1-thinking","place":30,"score":17.6,"version":"GPT-5.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":31,"score":16.25,"version":"Kimi K2.5"},{"company":"xai","family":"Grok","model":"grok-4","place":32,"score":16,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":33,"score":16,"version":"Grok 4"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":34,"score":14.8,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-32k","place":35,"score":13.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":36,"score":13.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":37,"score":13.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":38,"score":13.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":39,"score":13.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":40,"score":9.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":41,"score":9.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514-thinking-16k","place":42,"score":8.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-medium","place":43,"score":7.5,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":44,"score":7.22,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-8k","place":45,"score":6.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-16k","place":46,"score":6.9,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":47,"score":6.7,"version":"Grok 4.1"},{"company":"openai","family":"o3","model":"o3","place":48,"score":6.5,"version":"o3"},{"company":"openai","family":"o3","model":"o3-high","place":49,"score":6.5,"version":"o3"},{"company":"openai","family":"o4","model":"o4-mini","place":50,"score":6.1,"version":"o4"},{"company":"openai","family":"o4","model":"o4-mini-high","place":51,"score":6.1,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-16k","place":52,"score":5.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking-1k","place":53,"score":5.8,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":54,"score":5,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":55,"score":5,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":56,"score":4.9,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3-pro","place":57,"score":4.9,"version":"o3"},{"company":"openai","family":"o3","model":"o3-pro-high","place":58,"score":4.9,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-thinking-32k","place":59,"score":4.9,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514-thinking-8k","place":60,"score":4.5,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":61,"score":4.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":62,"score":4.4,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":63,"score":4.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-thinking-16k","place":64,"score":4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-medium","place":65,"score":4,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-preview-low","place":66,"score":4,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":67,"score":4,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":68,"score":4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":69,"score":3.8,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":70,"score":3.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":71,"score":3.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-thinking-1k","place":72,"score":3.4,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3-mini","place":73,"score":3,"version":"o3"},{"company":"openai","family":"o3","model":"o3-mini-high","place":74,"score":3,"version":"o3"},{"company":"openai","family":"o3","model":"o3-medium","place":75,"score":3,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-thinking-8k","place":76,"score":2.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-high","place":77,"score":2.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":78,"score":2.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-24k","place":79,"score":2.5,"version":"Gemini 2.5"},{"company":"arcprize","family":"ARChitects","model":"architects","place":80,"score":2.5,"version":"ARChitects"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":81,"score":2.5,"version":"Gemini 2.5"},{"company":"openai","family":"o4","model":"o4-mini-medium","place":82,"score":2.4,"version":"o4"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":83,"score":2.3,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-1k","place":84,"score":2.2,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-8k","place":85,"score":2.1,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-8k","place":86,"score":2.1,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":87,"score":2.1,"version":"o3"},{"company":"openai","family":"o3","model":"o3-pro-low","place":88,"score":2.1,"version":"o3"},{"company":"openai","family":"o3","model":"o3-low","place":89,"score":2,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-thinking-16k","place":90,"score":2,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3-pro-medium","place":91,"score":1.9,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-low","place":92,"score":1.9,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":93,"score":1.7,"version":"Gemini 2.5"},{"company":"openai","family":"o4","model":"o4-mini-low","place":94,"score":1.7,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":95,"score":1.7,"version":"GPT-5"},{"company":"arcprize","family":"ARChitects","model":"icecuber","place":96,"score":1.6,"version":"ARChitects"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":97,"score":1.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":98,"score":1.3,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":99,"score":1.3,"version":"Gemini 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":100,"score":1.3,"version":"DeepSeek R1"},{"company":"openai","family":"Codex","model":"codex-mini-latest","place":101,"score":1.3,"version":"Codex"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":102,"score":1.3,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":103,"score":1.1,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking-8k","place":104,"score":0.9,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":105,"score":0.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-1k","place":106,"score":0.9,"version":"Claude 4"},{"company":"openai","family":"o1","model":"o1-mini","place":107,"score":0.8,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-5-mini-low","place":108,"score":0.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":109,"score":0.8,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":110,"score":0.8,"version":"GPT-4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking-16k","place":111,"score":0.7,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":112,"score":0.4,"version":"GPT-4.1"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":113,"score":0.4,"version":"Grok 3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking-1k","place":114,"score":0.4,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":115,"score":0,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4o","place":116,"score":0,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":117,"score":0,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":118,"score":0,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":119,"score":0,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Deep Think","top_model_family":"Gemini","top_model_id":"gemini-3-deep-think","top_model_version":"Gemini 3","top_open_source":0,"top_percent":84.6,"topic":"General reasoning","total_models":1613,"url":"https://arcprize.org/"},{"bench_models":11,"description":"Average accuracy across ARC-Easy and ARC-Challenge.","github":"https://github.com/allenai/ai2-arc","has_recent":0,"id":"arc-average","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ARC Average","paper":"","placements":[{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":1,"score":60.5,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":2,"score":59.9,"version":"SmolLM1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":3,"score":58.5,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":4,"score":51.7,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":5,"score":49.2,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":6,"score":46.2,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":7,"score":43.9,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":8,"score":43.9,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":9,"score":43.7,"version":"SmolLM1"},{"company":"meta","family":"Llama","model":"llama-1b","place":10,"score":41.6,"version":"Llama 1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":11,"score":37.3,"version":"SmolLM2"}],"stars":"","top_company":"huggingface","top_model":"SmolLM2 1.7B Pretrained","top_model_family":"SmolLM","top_model_id":"smollm2-1.7b-pretrained","top_model_version":"SmolLM2","top_open_source":1,"top_percent":60.5,"topic":"Science QA (average)","total_models":1613,"url":"https://github.com/allenai/ai2-arc"},{"bench_models":169,"description":"Hard subset of AI2 Reasoning Challenge; grade-school science.","github":"","has_recent":0,"id":"arc-challenge","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ARC-Challenge","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":1,"score":96.9,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":2,"score":96.9,"version":"Llama 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-3","place":3,"score":96.4,"version":"Claude 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":4,"score":96.2,"version":"Kimi K2"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":5,"score":96.1,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":6,"score":95.96,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":7,"score":95.93,"version":"LLaDA 2.0"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":8,"score":95.9,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":9,"score":95.81,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":10,"score":95.6,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":11,"score":95.5,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":12,"score":95.16,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":13,"score":95.08,"version":"Ling 2.0"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":14,"score":94.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":15,"score":94.8,"version":"Llama 3.1"},{"company":"amazon","family":"Nova","model":"nova-pro","place":16,"score":94.8,"version":"Nova"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":17,"score":94.45,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":18,"score":94.4,"version":"Llama 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":19,"score":93.9,"version":"LLaDA 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":20,"score":93.54,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":21,"score":93.43,"version":"Mistral 3.2"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet","place":22,"score":93.2,"version":"Claude 3"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":23,"score":93.1,"version":"Llama 3"},{"company":"ai21","family":"Jamba","model":"jamba-1.5-large","place":24,"score":93,"version":"Jamba 1.5"},{"company":"tencent","family":"WeDLM","model":"wedlm-8b-instruct","place":25,"score":92.92,"version":"WeDLM"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":26,"score":92.9,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":27,"score":92.55,"version":"Qwen3"},{"company":"amazon","family":"Nova","model":"nova-lite","place":28,"score":92.4,"version":"Nova"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":29,"score":92.33,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":30,"score":92.06,"version":"OLMo 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":31,"score":91.89,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-instruct","place":32,"score":91.47,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3-24b-base","place":33,"score":91.3,"version":"Mistral 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":34,"score":91.3,"version":"Mixtral"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":35,"score":91,"version":"Phi 3.5 MoE"},{"company":"amazon","family":"Nova","model":"nova-micro","place":36,"score":90.2,"version":"Nova"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":37,"score":90.13,"version":"Llama 3.3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":38,"score":90.09,"version":"Gemma 3"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":39,"score":89.76,"version":"EuroLLM"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":40,"score":89.22,"version":"Mistral 3.2"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":41,"score":89.2,"version":"Claude 3"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":42,"score":86,"version":"Llama 2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":43,"score":85.9,"version":"Mixtral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":44,"score":85.8,"version":"Mixtral"},{"company":"ai21","family":"Jamba","model":"jamba-1.5-mini","place":45,"score":85.7,"version":"Jamba 1.5"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":46,"score":85.1,"version":"Llama 2"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":47,"score":84.67,"version":"Apertus"},{"company":"microsoft","family":"Phi","model":"phi-3.5-mini-instruct","place":48,"score":84.6,"version":"Phi 3.5"},{"company":"microsoft","family":"Phi","model":"phi-4-mini","place":49,"score":83.7,"version":"Phi 4"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":50,"score":83.4,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":51,"score":83.4,"version":"Llama 3.1"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":52,"score":82.66,"version":"EuroLLM"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":53,"score":82.4,"version":"Llama 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":54,"score":81.09,"version":"OLMo 2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":55,"score":79.7,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":56,"score":79.4,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":57,"score":78.6,"version":"Llama 3.2"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":58,"score":78.59,"version":"Apertus"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":59,"score":78.1,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":60,"score":77.2,"version":"Mistral"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":61,"score":74.06,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":62,"score":72.44,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":63,"score":72.4,"version":"Qwen2.5"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":64,"score":71.9,"version":"Ministral"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":65,"score":71.4,"version":"Gemma 2"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":66,"score":71.08,"version":"Solar Mini"},{"company":"cohere","family":"Command","model":"command-r-plus","place":67,"score":71,"version":"Command"},{"company":"cohere","family":"Command","model":"command-r-plus-08-2024","place":68,"score":71,"version":"Command"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":69,"score":70.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":70,"score":70.5,"version":"Qwen2.5"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":71,"score":70.5,"version":"HY"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":72,"score":70.39,"version":"Llama 3.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":73,"score":69.6,"version":"Motif 2"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":74,"score":69.2,"version":"Llama 3.1 Nemotron"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":75,"score":69.2,"version":"Llama 3.0"},{"company":"qwen","family":"Qwen","model":"qwen-2-72b-instruct","place":76,"score":68.9,"version":"Qwen2"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":77,"score":68.9,"version":"Gemma 3"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":78,"score":68.86,"version":"Solar Pro"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":79,"score":68.83,"version":"HY"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":80,"score":68.4,"version":"Gemma 2"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":81,"score":68.34,"version":"Phi 3.5 MoE"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":82,"score":67.32,"version":"Phi 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-14b-instruct","place":83,"score":67.3,"version":"Qwen2.5"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":84,"score":66.55,"version":"Phi 3"},{"company":"cohere","family":"Command","model":"command-r-08-2024","place":85,"score":66.5,"version":"Command"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":86,"score":66.4,"version":"Ouro"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":87,"score":66.3,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":88,"score":66.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":89,"score":66.04,"version":"Qwen1.5"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":90,"score":65.8,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":91,"score":65.7,"version":"Marin 32B"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-tfree-hat-pretrained-7b-dpo","place":92,"score":65.5,"version":"Llama TFree HAT"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":93,"score":65.44,"version":"Trinity Large"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":94,"score":65.4,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":95,"score":65.4,"version":"OLMo 3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":96,"score":64.9,"version":"K2 V2"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":97,"score":64.9,"version":"Llama 3.1"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5-0106-gemma","place":98,"score":64.68,"version":"OpenChat 3.5 0106"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0-instruct","place":99,"score":64.59,"version":"Bielik v3.0"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":100,"score":64.45,"version":"HY"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":101,"score":63.65,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":102,"score":63.65,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":103,"score":63.65,"version":"Qwen2.5"},{"company":"radicalnumerics","family":"RND1","model":"rnd1-base-0910","place":104,"score":63.2,"version":"RND1"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":105,"score":63.14,"version":"Mistral v0.2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":106,"score":62.7,"version":"MobileLLM"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.6-instruct","place":107,"score":62.54,"version":"Bielik v2.6"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":108,"score":62.37,"version":"Trinity Large"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.5-instruct","place":109,"score":61.95,"version":"Bielik v2.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":110,"score":61.9,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":111,"score":61.6,"version":"Gemma 3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0","place":112,"score":61.43,"version":"Bielik v3.0"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":113,"score":61.09,"version":"gemma-7b"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":114,"score":61.01,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":115,"score":60.92,"version":"Gemma 3"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":116,"score":60.92,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":117,"score":60.9,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.2","place":118,"score":60.84,"version":"Mistral v0.2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":119,"score":60.75,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"meta-llama-3-8b-instruct","place":120,"score":60.75,"version":"Llama 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":121,"score":60.6,"version":"Nemotron Nano v2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2","place":122,"score":60.58,"version":"Bielik v2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":123,"score":60.24,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":124,"score":60.24,"version":"Llama 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":125,"score":59.98,"version":"Falcon H1"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.1","place":126,"score":59.98,"version":"Mistral v0.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.2-instruct","place":127,"score":59.9,"version":"Bielik v2.2"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":128,"score":59.8,"version":"Dream 7B"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":129,"score":59.7,"version":"Mixtral"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.1-instruct","place":130,"score":59.56,"version":"Bielik v2.1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":131,"score":59.4,"version":"Llama 3.2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.3-instruct","place":132,"score":59.3,"version":"Bielik v2.3"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b-chat","place":133,"score":58.7,"version":"Qwen1.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.0-instruct","place":134,"score":58.62,"version":"Bielik v2.0"},{"company":"qwen","family":"Qwen","model":"qwen-14b","place":135,"score":58.28,"version":"Qwen"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":136,"score":57.8,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b","place":137,"score":56.57,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":138,"score":56.5,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":139,"score":56.14,"version":"Llama 4"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3-instruct","place":140,"score":56.06,"version":"Bielik v3"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":141,"score":56,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":142,"score":55.9,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-7b-chat","place":143,"score":55.89,"version":"Qwen1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":144,"score":55.72,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":145,"score":55.7,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":146,"score":55.5,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":147,"score":55.46,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":148,"score":54.9,"version":"Mistral"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":149,"score":54.52,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.1","place":150,"score":54.52,"version":"Mistral v0.1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":151,"score":54.44,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":152,"score":54.44,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":153,"score":54.35,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":154,"score":53.2,"version":"Gemma 2"},{"company":"google","family":"PaLM","model":"palm-540b","place":155,"score":53,"version":"PaLM"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":156,"score":52.7,"version":"Llama 1"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":157,"score":52.62,"version":"MobileLLM P1"},{"company":"google","family":"PaLM","model":"palm-62b","place":158,"score":52.5,"version":"PaLM"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":159,"score":52.47,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":160,"score":52.39,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":161,"score":51.96,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-3","place":162,"score":51.4,"version":"GPT-3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3","place":163,"score":51.19,"version":"Bielik v3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-base","place":164,"score":50.8,"version":"Granite 3.3"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":165,"score":49.89,"version":"HY"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":166,"score":49.57,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":167,"score":48.98,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":168,"score":48.8,"version":"Llama 2"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":169,"score":48.21,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":170,"score":47.6,"version":"Llama 1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-instruct-v0.1","place":171,"score":47.53,"version":"Bielik v0.1"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":172,"score":47.5,"version":"LLaDA"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":173,"score":46.59,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-v0.1","place":174,"score":45.22,"version":"Bielik v0.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":175,"score":44.88,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":176,"score":44.88,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":177,"score":44.54,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":178,"score":43.86,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":179,"score":43.77,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":180,"score":43.2,"version":"Llama 2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":181,"score":43.09,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":182,"score":42.1,"version":"Gemma 2"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":183,"score":42.06,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":184,"score":42.06,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":185,"score":41.98,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":186,"score":41.38,"version":"Qwen2.5"},{"company":"loka","family":"TRLM","model":"trlm-135m","place":187,"score":40.61,"version":"TRLM"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":188,"score":40.53,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":189,"score":39.25,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":190,"score":38.4,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":191,"score":38.28,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":192,"score":37.8,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":193,"score":37.71,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":194,"score":34.81,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":195,"score":34.64,"version":"Llama 3.2"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":196,"score":34.5,"version":"CodeLlama-7B"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":197,"score":34.13,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":198,"score":33.28,"version":"Qwen2.5"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":199,"score":33.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":200,"score":33.1,"version":"Pythia"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":201,"score":31.06,"version":"Qwen3"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":202,"score":30.1,"version":"Pythia"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":203,"score":29,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":204,"score":28.2,"version":"Gemma 3"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":205,"score":27.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":206,"score":24.5,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":207,"score":20.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":208,"score":20.1,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":209,"score":16.2,"version":"Pythia"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":210,"score":0.741,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":211,"score":0.739,"version":"Llama TFree HAT"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B","top_model_family":"Llama","top_model_id":"llama-3.1-405b","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":96.9,"topic":"Science QA","total_models":1613,"url":"https://github.com/allenai/ai2-arc"},{"bench_models":2,"description":"German translation of the ARC Challenge benchmark.","github":"","has_recent":0,"id":"arc-challenge-de","leaderboard":"","lower_is_better":false,"metric":"norm_log_acc","name":"ARC-Challenge (DE)","paper":"","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":0.669,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":2,"score":0.653,"version":"Llama 3.3"}],"stars":"","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":0.669,"topic":"Science QA (German)","total_models":1613,"url":""},{"bench_models":38,"description":"Easier subset of AI2 Reasoning Challenge.","github":"","has_recent":0,"id":"arc-easy","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ARC-Easy","paper":"","placements":[{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":1,"score":89,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":2,"score":89,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":3,"score":88.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":4,"score":88.2,"version":"Gemma 3"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":5,"score":88,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":6,"score":87.8,"version":"Marin 32B"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":7,"score":85.9,"version":"OLMo 2"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":8,"score":84.1,"version":"Motif 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":9,"score":83.6,"version":"Nemotron Nano v2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":10,"score":83.1,"version":"Mixtral"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":11,"score":81.5,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":12,"score":80.8,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":13,"score":80.5,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":14,"score":80,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":15,"score":80,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":16,"score":79.9,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":17,"score":78.9,"version":"Llama 1"},{"company":"google","family":"PaLM","model":"palm-540b","place":18,"score":76.6,"version":"PaLM"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":19,"score":76.28,"version":"MobileLLM P1"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":20,"score":75.2,"version":"Llama 2"},{"company":"google","family":"PaLM","model":"palm-62b","place":21,"score":75.2,"version":"PaLM"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":22,"score":74.8,"version":"Llama 1"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":23,"score":73.2,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":24,"score":73,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":25,"score":72.8,"version":"Llama 1"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":26,"score":70.8,"version":"Pythia"},{"company":"openai","family":"GPT","model":"gpt-3","place":27,"score":68.8,"version":"GPT-3"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":28,"score":68.7,"version":"Llama 2"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":29,"score":68.6,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":30,"score":63.5,"version":"Pythia"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":31,"score":63.47,"version":"Llama 3.2"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":32,"score":61.7,"version":"Pythia"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":33,"score":59.4,"version":"CodeLlama-7B"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":34,"score":58.5,"version":"Pythia"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":35,"score":57.7,"version":"Gemma 3"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":36,"score":51.7,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":37,"score":44,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":38,"score":38.5,"version":"Pythia"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":39,"score":0.92,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":40,"score":0.911,"version":"Llama 3.3"}],"stars":"","top_company":"google","top_model":"Gemma 3 PT 27B","top_model_family":"Gemma","top_model_id":"gemma-3-pt-27b","top_model_version":"Gemma 3","top_open_source":1,"top_percent":89,"topic":"Science QA","total_models":1613,"url":"https://github.com/allenai/ai2-arc"},{"bench_models":2,"description":"German translation of the ARC Easy science QA benchmark.","github":"","has_recent":0,"id":"arc-easy-de","leaderboard":"","lower_is_better":false,"metric":"norm_log_acc","name":"ARC-Easy (DE)","paper":"","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":0.848,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":2,"score":0.825,"version":"Llama 3.3"}],"stars":"","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":0.848,"topic":"Science QA (German)","total_models":1613,"url":""},{"bench_models":36,"description":"Hard prompts on Chatbot Arena.","github":"","has_recent":0,"id":"arena-hard","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Arena-Hard","paper":"","placements":[{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":1,"score":97.1,"version":"Mistral 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":96.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":3,"score":95.7,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":4,"score":95.6,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":5,"score":95.1,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":6,"score":93.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":7,"score":93.2,"version":"DeepSeek R1"},{"company":"openai","family":"o1","model":"o1","place":8,"score":92.1,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":9,"score":92.1,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":10,"score":91.7,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":11,"score":89.5,"version":"Nemotron Cascade"},{"company":"openai","family":"o3","model":"o3-mini","place":12,"score":89,"version":"o3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":13,"score":87.9,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":14,"score":85.8,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":15,"score":85.8,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":16,"score":74.6,"version":"Nemotron Nano v2"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":17,"score":57.56,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":18,"score":55.25,"version":"Granite 3.2"},{"company":"ai21","family":"Jamba","model":"jamba-mini-1.6","place":19,"score":51.2,"version":"Jamba Mini"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":20,"score":46.48,"version":"Granite 4.0"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":21,"score":41.35,"version":"Ministral"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":22,"score":37.58,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":23,"score":36.43,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":24,"score":28.86,"version":"Granite 3.3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":25,"score":28.17,"version":"Llama 3.1"},{"company":"cohere","family":"Command","model":"command-r7b","place":26,"score":27.95,"version":"Command"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":27,"score":26.7,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":28,"score":25.84,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":29,"score":25.52,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":30,"score":25.44,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":31,"score":24.86,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":32,"score":24.19,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":33,"score":23.3,"version":"Granite 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":34,"score":17.17,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":35,"score":10.36,"version":"DeepSeek R1"},{"company":"xai","family":"Grok","model":"grok-3-beta-think","place":36,"score":null,"version":"Grok 3"}],"stars":"920","top_company":"mistral","top_model":"Mistral Medium 3","top_model_family":"Mistral","top_model_id":"mistral-medium-3","top_model_version":"Mistral 3","top_open_source":0,"top_percent":97.1,"topic":"Chat ability","total_models":1613,"url":"https://github.com/lm-sys/arena-hard-auto"},{"bench_models":78,"description":"Updated Arena-Hard v2 prompts on Chatbot Arena.","github":"https://github.com/lmarena/arena-hard-auto","has_recent":0,"id":"arena-hard-v2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Arena-Hard V2","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":1,"score":90.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-moe-2507","place":2,"score":88.2,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":3,"score":87,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":4,"score":86.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-thinking-high","place":5,"score":82.91,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":6,"score":82.7,"version":"Qwen3 Next"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16-high","place":7,"score":81.7,"version":"o4"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":81.7,"version":"Gemini 3"},{"company":"antgroup","family":"Ring","model":"ring-1t","place":9,"score":81.59,"version":"Ring"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":10,"score":80.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":11,"score":80.18,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":12,"score":79.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":13,"score":79.6,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":14,"score":79.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":15,"score":79.2,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":16,"score":78.66,"version":"Kimi K2.5"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":17,"score":78,"version":"o4"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":18,"score":77.92,"version":"Ling 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":19,"score":76.7,"version":"Claude 4.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":20,"score":74.85,"version":"Ling 1T"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":21,"score":74.8,"version":"Solar Open"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":22,"score":73.41,"version":"Ring"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":23,"score":73.2,"version":"Nanbeige4.1"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":24,"score":72.7,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":25,"score":72.54,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":26,"score":71.04,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":27,"score":69,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":28,"score":67.7,"version":"Nemotron 3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":29,"score":67.7,"version":"Nemotron 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":30,"score":66.5,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":31,"score":66.1,"version":"Kimi K2"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-high","place":32,"score":64.8,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":33,"score":62.5,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":34,"score":62.3,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":35,"score":61.5,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":36,"score":61.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":37,"score":60.27,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":38,"score":60.2,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":39,"score":60,"version":"Nanbeige4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":40,"score":60,"version":"Qwen3"},{"company":"openai","family":"o1","model":"o1-2024-12-17-high","place":41,"score":58.7,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":42,"score":58.5,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":43,"score":58.3,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":44,"score":57.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":45,"score":57.8,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":46,"score":56.7,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":47,"score":56.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":48,"score":56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":49,"score":56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":50,"score":55.1,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":51,"score":51.5,"version":"Claude 4"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":52,"score":50.2,"version":"o1"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":53,"score":50,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5","place":54,"score":49.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":55,"score":48.6,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":56,"score":48.6,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-a4b","place":57,"score":48.5,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":58,"score":48.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":59,"score":48.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":60,"score":48,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3-7-sonnet-20250219-thinking-16k","place":61,"score":47,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":62,"score":46.7,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":63,"score":45.1,"version":"Gemini 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":64,"score":43.1,"version":"Mistral 3.2"},{"company":"openai","family":"GPT","model":"gpt-4.5-preview","place":65,"score":43,"version":"GPT-4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":66,"score":40.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":67,"score":39.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":68,"score":36.9,"version":"Qwen3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":69,"score":36.1,"version":"QwQ"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":70,"score":35.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":71,"score":34.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":72,"score":34.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":73,"score":28.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":74,"score":26.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":75,"score":26.3,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20241022","place":76,"score":25.8,"version":"Claude 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":77,"score":19.56,"version":"Mistral 3.1"},{"company":"scale","family":"s1","model":"s1.1-32b","place":78,"score":18.3,"version":"s1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl-thinking","place":79,"score":16.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":80,"score":15.4,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking","place":81,"score":13.7,"version":"Qwen3"},{"company":"nvidia","family":"Athene","model":"athene-v2-chat","place":82,"score":12.6,"version":"Athene V2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":83,"score":12.6,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama4-maverick-instruct-basic","place":84,"score":12,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl","place":85,"score":10.7,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":86,"score":9.7,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":87,"score":9.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":88,"score":8,"version":"Qwen2.5"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-70b-instruct-hf","place":89,"score":6.8,"version":"Llama 3.1"},{"company":"openthinker","family":"OpenThinker","model":"openthinker2-32b","place":90,"score":2.3,"version":"OpenThinker2"}],"stars":"920","top_company":"qwen","top_model":"Qwen3 Max Thinking","top_model_family":"Qwen","top_model_id":"qwen3-max-thinking","top_model_version":"Qwen3","top_open_source":0,"top_percent":90.2,"topic":"Chat ability","total_models":1613,"url":"https://github.com/lm-sys/arena-hard-auto"},{"bench_models":16,"description":"Chatbot Arena Hard V2 creative writing win-rate subset.","github":"","has_recent":0,"id":"arena-hard-v2-creative-writing","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Arena-Hard V2 Creative Writing","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":93.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":92.2,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":3,"score":88.8,"version":"DeepSeek V3.2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":4,"score":86.2,"version":"MiMo V2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":5,"score":86.2,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":6,"score":85,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":7,"score":80.1,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":8,"score":76.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":9,"score":72.8,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":10,"score":68.5,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":11,"score":66,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":12,"score":63.2,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":13,"score":61.5,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":14,"score":59.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":15,"score":59.3,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":16,"score":54.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":17,"score":25.9,"version":"GPT-OSS"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":93.6,"topic":"Creative writing","total_models":1613,"url":"https://github.com/lm-sys/arena-hard-auto"},{"bench_models":16,"description":"Chatbot Arena Hard V2 benchmark using the hard prompt win-rate subset.","github":"","has_recent":0,"id":"arena-hard-v2-hard-prompt","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Arena-Hard V2 Hard Prompt","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":72.6,"version":"Gemini 3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":2,"score":72.1,"version":"Nemotron 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":3,"score":71.9,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":71.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":5,"score":71.2,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":6,"score":63.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":7,"score":59.7,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":8,"score":54.5,"version":"Kimi K2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":9,"score":54.1,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":10,"score":53.4,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":11,"score":51.7,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":12,"score":51.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":13,"score":49.6,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":14,"score":48.7,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":15,"score":39.9,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":16,"score":39.9,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":72.6,"topic":"Chat ability","total_models":1613,"url":"https://github.com/lm-sys/arena-hard-auto"},{"bench_models":11,"description":"ARKitScenes benchmark for assessing 3D scene reconstruction and understanding from mixed reality captures.","github":"","has_recent":0,"id":"arkitscenes","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ARKitScenes","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":1,"score":61.5,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":2,"score":56.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":3,"score":56.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":4,"score":56.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":5,"score":56.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":6,"score":55.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":7,"score":53.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":8,"score":45.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":9,"score":36.2,"version":"Claude 4"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":10,"score":30.3,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":7.4,"version":"Gemini 2.5"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5-VL 72B Instruct","top_model_family":"Qwen","top_model_id":"qwen2.5-vl-72b-instruct","top_model_version":"Qwen2.5 VL","top_open_source":1,"top_percent":61.5,"topic":"3D scene understanding","total_models":1613,"url":""},{"bench_models":39,"description":"Evaluation suite for adversarial red-teaming of autonomous AI agents.","github":"","has_recent":0,"id":"art-agent-red-teaming","leaderboard":"","lower_is_better":true,"metric":"percent","name":"ART Agent Red Teaming","paper":"https://arxiv.org/abs/2507.20526","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":33.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":2,"score":40,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":40,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":44.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":5,"score":51.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":6,"score":53.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":7,"score":54.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":8,"score":56.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":9,"score":57.8,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":60.7,"version":"Gemini 3"},{"company":"openai","family":"o3","model":"o3","place":11,"score":62.7,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":12,"score":63.6,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":13,"score":64.5,"version":"Claude 3.7"},{"company":"xai","family":"Grok","model":"grok-4","place":14,"score":65,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":15,"score":65.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":16,"score":65.9,"version":"Claude 4.1"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":17,"score":70.9,"version":"Gemini 3"},{"company":"openai","family":"o1","model":"o1","place":18,"score":71.8,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":19,"score":75,"version":"Claude 3.5"},{"company":"openai","family":"o4","model":"o4-mini","place":20,"score":75.5,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":21,"score":76.1,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":22,"score":76.4,"version":"Claude 3.5"},{"company":"cohere","family":"Command","model":"command-r","place":23,"score":76.4,"version":"Command"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":24,"score":77.5,"version":"Grok 4.1"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":25,"score":80.5,"version":"GPT-4.5"},{"company":"xai","family":"Grok","model":"grok-2","place":26,"score":82.7,"version":"Grok 2"},{"company":"openai","family":"o3","model":"o3-mini","place":27,"score":84.5,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-05-06","place":28,"score":85,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":29,"score":85,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":30,"score":85.5,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":31,"score":86.4,"version":"GPT-4o"},{"company":"openai","family":"o3","model":"o3-mini-high","place":32,"score":86.4,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":33,"score":87.7,"version":"Gemini 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":34,"score":89,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":35,"score":90.6,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":36,"score":90.9,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":37,"score":91.7,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":38,"score":92.2,"version":"Llama 3.3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":39,"score":93.6,"version":"Gemini 2.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":33.6,"topic":"Agent robustness","total_models":1613,"url":""},{"bench_models":32,"description":"Artifacts-focused coding and tool-use benchmark evaluating generated code artifacts.","github":"https://github.com/Tencent-Hunyuan/ArtifactsBenchmark","has_recent":0,"id":"artifactsbench","leaderboard":"https://artifactsbenchmark.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"ArtifactsBench","paper":"https://arxiv.org/abs/2507.04952","placements":[{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":1,"score":73,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":72.55,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":3,"score":66.8,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":4,"score":61.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":60.28,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":6,"score":59.8,"version":"GLM 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":7,"score":59.76,"version":"Claude 4.1"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":8,"score":59.31,"version":"Ling 1T"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":57.74,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":57.7,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":11,"score":57.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":12,"score":57.28,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":13,"score":56.91,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":14,"score":55.8,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":15,"score":55.01,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":16,"score":54.2,"version":"Kimi K2"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":17,"score":54.04,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":18,"score":51.33,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":19,"score":51.32,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":20,"score":50.62,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":21,"score":48.9,"version":"GLM 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":22,"score":47.73,"version":"DeepSeek R1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":23,"score":47.65,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":24,"score":47.15,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":25,"score":45.95,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":26,"score":44.87,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":27,"score":43.5,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":28,"score":43.29,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":29,"score":41.41,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":30,"score":41.09,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5","place":31,"score":41.04,"version":"GPT-5"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-80b-a13b-instruct","place":32,"score":40.95,"version":"Hunyuan"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":33,"score":39.85,"version":"Claude 3.5"},{"company":"kuaishou","family":"KAT","model":"kat-v1-40b","place":34,"score":35.21,"version":"KAT V1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":35,"score":33.54,"version":"GPT-4o"}],"stars":"","top_company":"openai","top_model":"GPT-5 Thinking","top_model_family":"GPT","top_model_id":"gpt-5-thinking","top_model_version":"GPT-5","top_open_source":0,"top_percent":73,"topic":"Agentic coding","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on AMI meeting speech.","github":"","has_recent":0,"id":"asr-ami","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR AMI","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":1,"score":15.05,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":2,"score":15.36,"version":"LFM2-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":3,"score":15.63,"version":"LFM2.5-Audio"},{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":4,"score":15.95,"version":"Whisper v3"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5-Omni-3B","top_model_family":"Qwen","top_model_id":"qwen2.5-omni-3b","top_model_version":"Qwen2.5 Omni","top_open_source":1,"top_percent":15.05,"topic":"ASR","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on Earnings22 financial calls.","github":"","has_recent":0,"id":"asr-earnings22","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR Earnings22","paper":"","placements":[{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":1,"score":11.29,"version":"Whisper v3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":2,"score":14.56,"version":"LFM2.5-Audio"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":3,"score":14.81,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":4,"score":19.75,"version":"LFM2-Audio"}],"stars":"","top_company":"openai","top_model":"Whisper-large-V3","top_model_family":"Whisper","top_model_id":"whisper-large-v3","top_model_version":"Whisper v3","top_open_source":1,"top_percent":11.29,"topic":"ASR","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on GigaSpeech.","github":"","has_recent":0,"id":"asr-gigaspeech","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR GigaSpeech","paper":"","placements":[{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":1,"score":10.02,"version":"Whisper v3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":2,"score":10.47,"version":"LFM2.5-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":3,"score":10.63,"version":"LFM2-Audio"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":4,"score":11.76,"version":"Qwen2.5 Omni"}],"stars":"","top_company":"openai","top_model":"Whisper-large-V3","top_model_family":"Whisper","top_model_id":"whisper-large-v3","top_model_version":"Whisper v3","top_open_source":1,"top_percent":10.02,"topic":"ASR","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on LibriSpeech clean split.","github":"","has_recent":0,"id":"asr-librispeech-clean","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR LibriSpeech Clean","paper":"","placements":[{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":1,"score":1.95,"version":"LFM2.5-Audio"},{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":2,"score":2.01,"version":"Whisper v3"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":3,"score":2.03,"version":"LFM2-Audio"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":4,"score":2.14,"version":"Qwen2.5 Omni"}],"stars":"","top_company":"liquidai","top_model":"LFM2.5-Audio-1.5B","top_model_family":"LFM","top_model_id":"lfm2.5-audio-1.5b","top_model_version":"LFM2.5-Audio","top_open_source":1,"top_percent":1.95,"topic":"ASR","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on LibriSpeech other split.","github":"","has_recent":0,"id":"asr-librispeech-other","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR LibriSpeech Other","paper":"","placements":[{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":1,"score":3.91,"version":"Whisper v3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":2,"score":4.3,"version":"LFM2.5-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":3,"score":4.39,"version":"LFM2-Audio"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":4,"score":4.52,"version":"Qwen2.5 Omni"}],"stars":"","top_company":"openai","top_model":"Whisper-large-V3","top_model_family":"Whisper","top_model_id":"whisper-large-v3","top_model_version":"Whisper v3","top_open_source":1,"top_percent":3.91,"topic":"ASR","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on SPGISpeech.","github":"","has_recent":0,"id":"asr-spgispeech","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR SPGISpeech","paper":"","placements":[{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":1,"score":2.76,"version":"LFM2.5-Audio"},{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":2,"score":2.94,"version":"Whisper v3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":3,"score":3.24,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":4,"score":4.17,"version":"LFM2-Audio"}],"stars":"","top_company":"liquidai","top_model":"LFM2.5-Audio-1.5B","top_model_family":"LFM","top_model_id":"lfm2.5-audio-1.5b","top_model_version":"LFM2.5-Audio","top_open_source":1,"top_percent":2.76,"topic":"ASR","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on TED-LIUM.","github":"","has_recent":0,"id":"asr-ted-lium","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR TED-LIUM","paper":"","placements":[{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":1,"score":3.47,"version":"LFM2.5-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":2,"score":3.56,"version":"LFM2-Audio"},{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":3,"score":3.86,"version":"Whisper v3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":4,"score":5.08,"version":"Qwen2.5 Omni"}],"stars":"","top_company":"liquidai","top_model":"LFM2.5-Audio-1.5B","top_model_family":"LFM","top_model_id":"lfm2.5-audio-1.5b","top_model_version":"LFM2.5-Audio","top_open_source":1,"top_percent":3.47,"topic":"ASR","total_models":1613,"url":""},{"bench_models":4,"description":"Automatic speech recognition benchmark on VoxPopuli.","github":"","has_recent":0,"id":"asr-voxpopuli","leaderboard":"","lower_is_better":true,"metric":"wer","name":"ASR VoxPopuli","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":1,"score":5.6,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":2,"score":7.13,"version":"LFM2.5-Audio"},{"company":"openai","family":"Whisper","model":"whisper-large-v3","place":3,"score":9.54,"version":"Whisper v3"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":4,"score":9.93,"version":"LFM2-Audio"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5-Omni-3B","top_model_family":"Qwen","top_model_id":"qwen2.5-omni-3b","top_model_version":"Qwen2.5 Omni","top_open_source":1,"top_percent":5.6,"topic":"ASR","total_models":1613,"url":""},{"bench_models":21,"description":"Evaluates science agents across literature understanding, data analysis, planning, tool use, coding, and search.","github":"","has_recent":0,"id":"astabench","leaderboard":"https://huggingface.co/spaces/allenai/asta-bench-leaderboard","lower_is_better":false,"metric":"percent","name":"AstaBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":1,"score":53,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":43.3,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":3,"score":39.8,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":4,"score":33,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":5,"score":31.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":6,"score":25.7,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":7,"score":20.3,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":8,"score":16.2,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4o","place":9,"score":16,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":10,"score":12.4,"version":"Llama 4"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":11,"score":2.6,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":12,"score":2.4,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":13,"score":2.3,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":14,"score":2.2,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-2024-05","place":15,"score":2.1,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-5-haiku-20241022","place":16,"score":1.9,"version":"Claude 3.5"},{"company":"you.com","family":"You.com","model":"youcom-search-api","place":17,"score":1.6,"version":"You.com"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":18,"score":1.6,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-2025-08","place":19,"score":1.1,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-2025-08-07","place":20,"score":1.1,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":21,"score":0.5,"version":"GPT-4o"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4","top_model_family":"Claude","top_model_id":"claude-sonnet-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":53,"topic":"Agent evaluation","total_models":1613,"url":""},{"bench_models":15,"description":"Adversarial jailbreak suite measuring refusal robustness against targeted attack prompts.","github":"","has_recent":0,"id":"attaq","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AttaQ","paper":"","placements":[{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":1,"score":88.5,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":2,"score":87.47,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":3,"score":86.64,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":4,"score":86.1,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":5,"score":86.05,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":6,"score":85.73,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":7,"score":84.7,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":8,"score":84.7,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":9,"score":83.72,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":10,"score":83.72,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":11,"score":83.43,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":12,"score":83.23,"version":"Granite 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":13,"score":81.9,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":14,"score":42.87,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":15,"score":42.45,"version":"DeepSeek R1"}],"stars":"","top_company":"IBM","top_model":"Granite 3.3 8B Instruct","top_model_family":"Granite","top_model_id":"granite-3.3-8b-instruct","top_model_version":"Granite 3.3","top_open_source":1,"top_percent":88.5,"topic":"Safety / jailbreak","total_models":1613,"url":""},{"bench_models":40,"description":"End-to-end autonomous coding benchmark with unit-test based execution across diverse repositories and tasks.","github":"https://github.com/Tencent-Hunyuan/AutoCodeBenchmark","has_recent":0,"id":"autocodebench","leaderboard":"https://autocodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"AutoCodeBench","paper":"https://arxiv.org/abs/2508.09101","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4-thinking","place":1,"score":52.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":2,"score":51.1,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-high","place":3,"score":51.1,"version":"o3"},{"company":"xai","family":"Grok","model":"grok-4","place":4,"score":50.9,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":5,"score":50.9,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":6,"score":50.2,"version":"DeepSeek R1"},{"company":"openai","family":"o4","model":"o4-mini","place":7,"score":50,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":8,"score":49.3,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":48.7,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":10,"score":48.1,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":11,"score":48,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":12,"score":47.8,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":13,"score":47.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":14,"score":46.6,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":15,"score":45.9,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":16,"score":45.7,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":17,"score":45.3,"version":"Seed 1.6"},{"company":"bytedance","family":"Seed","model":"seed1.6-thinking","place":18,"score":45,"version":"Seed 1.6"},{"company":"qwen","family":"Qwen","model":"qwen-3-coder-480b-a35b-instruct","place":19,"score":44.8,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.6-thinking","place":20,"score":44.7,"version":"Seed 1.6"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-turbos","place":21,"score":43.8,"version":"Hunyuan Turbos"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":22,"score":43.1,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":23,"score":42.9,"version":"Seed 1.6"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":24,"score":41.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":25,"score":41.1,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":26,"score":40.8,"version":"GLM 4.5"},{"company":"baidu","family":"ERNIE","model":"ernie-x1","place":27,"score":39.6,"version":"ERNIE X1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":28,"score":37.7,"version":"DeepSeek Coder V2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":29,"score":37.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":30,"score":35.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":31,"score":34.3,"version":"Qwen2.5"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-coder-b8-preview","place":32,"score":33.4,"version":"Hunyuan"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":33,"score":32.3,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":34,"score":31,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":35,"score":28.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":36,"score":28.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":37,"score":28.5,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking","place":38,"score":24.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":39,"score":23.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":40,"score":22.5,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":41,"score":20.5,"version":"DeepSeek Coder"},{"company":"map","family":"OpenCoder","model":"opencoder-8b-instruct","place":42,"score":19.3,"version":"OpenCoder"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4 (Thinking)","top_model_family":"Claude","top_model_id":"claude-opus-4-thinking","top_model_version":"Claude 4","top_open_source":0,"top_percent":52.4,"topic":"Autonomous coding","total_models":1613,"url":"https://autocodebench.github.io/"},{"bench_models":10,"description":"Lite version of AutoCodeBench focusing on smaller tasks with the same end-to-end, unit-test-based evaluation.","github":"https://github.com/Tencent-Hunyuan/AutoCodeBenchmark","has_recent":0,"id":"autocodebench-lite","leaderboard":"https://autocodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"AutoCodeBench-Lite","paper":"https://arxiv.org/abs/2508.09101","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":1,"score":64.5,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-high","place":2,"score":63.2,"version":"o3"},{"company":"xai","family":"Grok","model":"grok-4","place":3,"score":63,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":4,"score":62,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":5,"score":61.5,"version":"DeepSeek R1"},{"company":"openai","family":"o4","model":"o4-mini","place":6,"score":60.5,"version":"o4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":59.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":8,"score":57.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":9,"score":56.9,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":10,"score":56.8,"version":"DeepSeek V3"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4","top_model_family":"Claude","top_model_id":"claude-opus-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":64.5,"topic":"Autonomous coding","total_models":1613,"url":"https://autocodebench.github.io/"},{"bench_models":7,"description":"AutoLogi benchmark evaluating automated logical reasoning accuracy.","github":"","has_recent":0,"id":"autologi","leaderboard":"","lower_is_better":false,"metric":"percent","name":"AutoLogi","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":1,"score":89.8,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":2,"score":89.5,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":3,"score":88.9,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":4,"score":88.2,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":5,"score":86.1,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":6,"score":84.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":83.3,"version":"Qwen3"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4","top_model_family":"Claude","top_model_id":"claude-sonnet-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":89.8,"topic":"Logical reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"STEM reasoning benchmark evaluating broad applied and basic engineering knowledge.","github":"","has_recent":0,"id":"babe","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BABE","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":58.1,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":55.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":51.3,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":4,"score":50.2,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":5,"score":50,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":49.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":7,"score":49.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":44.7,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":40.4,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":58.1,"topic":"STEM reasoning","total_models":1613,"url":""},{"bench_models":11,"description":"Visual reasoning benchmark testing basic visual perception and understanding.","github":"","has_recent":0,"id":"babyvision","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BabyVision","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":52.3,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":49.7,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":3,"score":44.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":40.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":5,"score":38.4,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":36.5,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":34.4,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":8,"score":22.2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":9,"score":20.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":10,"score":18.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":14.2,"version":"Claude 4.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":52.3,"topic":"Visual reasoning","total_models":1613,"url":""},{"bench_models":27,"description":"Benchmark for assessing LLM agents under adversarial and out-of-distribution tool-use scenarios.","github":"https://github.com/balrog-ai/BALROG","has_recent":0,"id":"balrog","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BALROG","paper":"https://arxiv.org/abs/2411.13543","placements":[{"company":"xai","family":"Grok","model":"grok-4","place":1,"score":43.6,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-exp-03.25","place":2,"score":43.3,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":3,"score":34.9,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":4,"score":33.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal-think","place":5,"score":32.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":6,"score":32.6,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-05-13","place":7,"score":32.3,"version":"GPT-4o"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":8,"score":29.5,"version":"Grok 3"},{"company":"reka","family":"Reka Flash","model":"reka-flash-3","place":9,"score":29.2,"version":"Reka Flash"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":10,"score":27.9,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-instruct","place":11,"score":27.3,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":12,"score":23,"version":"Llama 3.3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":13,"score":21,"version":"Gemini 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":14,"score":19.5,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku-20241022","place":15,"score":19.3,"version":"Claude 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-nemo-it-2407","place":16,"score":17.6,"version":"Mistral"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":17,"score":17.4,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.2-11b-instruct","place":18,"score":16.8,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":19,"score":16.2,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":20,"score":15.1,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":21,"score":14.6,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b-instruct","place":22,"score":12.8,"version":"Qwen2 VL"},{"company":"microsoft","family":"Phi","model":"phi-4","place":23,"score":11.6,"version":"Phi 3.5"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":24,"score":10.1,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":25,"score":7.8,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":26,"score":6.6,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b-instruct","place":27,"score":3.7,"version":"Qwen2 VL"}],"stars":"","top_company":"xai","top_model":"Grok 4","top_model_family":"Grok","top_model_id":"grok-4","top_model_version":"Grok 4","top_open_source":0,"top_percent":43.6,"topic":"Agent robustness","total_models":1613,"url":"https://balrogai.com/"},{"bench_models":177,"description":"Hard subset of BIG-bench with diverse reasoning tasks.","github":"","has_recent":0,"id":"bbh","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BBH","paper":"https://arxiv.org/abs/2210.09261","placements":[{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":1,"score":94.3,"version":"ERNIE 4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":2,"score":93.1,"version":"Claude 3.5"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":3,"score":92.08,"version":"Seed 1.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":4,"score":91.1,"version":"Qwen3 VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":5,"score":91,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":6,"score":90.6,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":7,"score":89.8,"version":"OLMo 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":8,"score":89.7,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":9,"score":89.5,"version":"DeepSeek V3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":10,"score":89.36,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":11,"score":89.3,"version":"Qwen2.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":12,"score":89.2,"version":"Gemini 1.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":13,"score":89.1,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":14,"score":88.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":15,"score":88.87,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":16,"score":88.8,"version":"OLMo 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":17,"score":88.8,"version":"Tulu 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":18,"score":88.7,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":19,"score":88.7,"version":"DeepSeek V3.2"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":20,"score":88.69,"version":"LLaDA 2.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":21,"score":88.6,"version":"Tulu 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":22,"score":88.5,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":23,"score":88.4,"version":"DeepSeek V3"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":24,"score":88.36,"version":"Seed 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":25,"score":88.2,"version":"DeepSeek V3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":26,"score":88,"version":"Tulu 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":27,"score":87.82,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":28,"score":87.8,"version":"Qwen3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":29,"score":87.7,"version":"Hermes 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":30,"score":87.6,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":31,"score":87.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":32,"score":87.38,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":33,"score":87.1,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":34,"score":87.1,"version":"Llama 3.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":35,"score":86.75,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":36,"score":86.3,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":37,"score":86.3,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":38,"score":86.22,"version":"DeepSeek V3"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":39,"score":86.2,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":40,"score":86.13,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":41,"score":85.9,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":42,"score":85.9,"version":"Llama 3.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":43,"score":85.78,"version":"Motif 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":44,"score":85.7,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":45,"score":85.54,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":46,"score":84.5,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":47,"score":84.1,"version":"GPT-4.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":48,"score":83.7,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":49,"score":83.68,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":50,"score":83.62,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":51,"score":83.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":52,"score":83.3,"version":"GPT-4o"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":53,"score":83.2,"version":"K2 V2"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet","place":54,"score":82.9,"version":"Claude 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":55,"score":82.7,"version":"Tulu 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":56,"score":82.68,"version":"MiniCPM 4.1"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":57,"score":82.4,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":58,"score":82.32,"version":"Llama 3.3"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":59,"score":82.2,"version":"Llama 3.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":60,"score":82.16,"version":"Gemma 3"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":61,"score":82.1,"version":"Llama 3.1"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":62,"score":82.1,"version":"Hermes 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":63,"score":82,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":64,"score":81.8,"version":"Tulu 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":65,"score":81.62,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":66,"score":81.6,"version":"Llama 3.1"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":67,"score":81.55,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":68,"score":81.54,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":69,"score":81.54,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":70,"score":81.5,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":71,"score":81.34,"version":"Motif 2"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":72,"score":81.3,"version":"Llama 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":73,"score":81.3,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":74,"score":81.07,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":75,"score":80.9,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":76,"score":80.58,"version":"LLaDA 2.1"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":77,"score":80.46,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":78,"score":79.97,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":79,"score":79.76,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":80,"score":79.48,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":81,"score":79.1,"version":"Phi 3.5 MoE"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":82,"score":79.08,"version":"HY"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":83,"score":79,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":84,"score":78.42,"version":"LLaDA 2.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":85,"score":78.41,"version":"Gemma 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":86,"score":78.34,"version":"Motif 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":87,"score":78.21,"version":"LLaDA 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":88,"score":78.13,"version":"Mistral 3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":89,"score":77.8,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":90,"score":77.7,"version":"Gemma 3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":91,"score":77.7,"version":"Mi:dm 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":92,"score":77.65,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":93,"score":77.6,"version":"OLMo 3"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":94,"score":77.5,"version":"Youtu-LLM"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":95,"score":76.46,"version":"Kanana 2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":96,"score":76.3,"version":"SmollM3"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":97,"score":75.54,"version":"HY"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":98,"score":74.8,"version":"HY"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":99,"score":74.28,"version":"Nemotron Nano V2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":100,"score":74.17,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":101,"score":73.8,"version":"Llama 3.1"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":102,"score":73.7,"version":"Claude 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":103,"score":72.6,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":104,"score":72.48,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":105,"score":72.2,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":106,"score":71.86,"version":"Granite 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":107,"score":71.56,"version":"Llama 3.1"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":108,"score":71.02,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":109,"score":70.95,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":110,"score":70.68,"version":"Falcon H1"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":111,"score":69.87,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":112,"score":69.54,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":113,"score":69.36,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":114,"score":69.15,"version":"Llama 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":115,"score":69.13,"version":"Granite 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":116,"score":69.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":117,"score":68.72,"version":"Qwen2.5"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":118,"score":68.4,"version":"K2 V2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":119,"score":67.9,"version":"Tulu 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":120,"score":67.79,"version":"OLMo 2"},{"company":"radicalnumerics","family":"RND1","model":"rnd1-base-0910","place":121,"score":67.5,"version":"RND1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":122,"score":67.4,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":123,"score":67.39,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":124,"score":67.38,"version":"DeepSeek R1"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":125,"score":67.31,"version":"Solar Pro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":126,"score":67.2,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":127,"score":66.34,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":128,"score":66.32,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":129,"score":66,"version":"Gemma 2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":130,"score":66,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":131,"score":65.8,"version":"Tulu 3"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":132,"score":65.7,"version":"Trinity Large"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":133,"score":65.6,"version":"OLMo 2"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":134,"score":64.9,"version":"Llama 4"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":135,"score":64.88,"version":"Gemma 2"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":136,"score":64.39,"version":"Ministral 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":137,"score":64.2,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":138,"score":63.91,"version":"Granite 3.3"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":139,"score":63.17,"version":"Falcon H1R"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":140,"score":63.09,"version":"Phi 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":141,"score":62.8,"version":"Llama 3.1"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":142,"score":62.58,"version":"Phi 3.5 MoE"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":143,"score":62.47,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":144,"score":62.28,"version":"Falcon H1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":145,"score":62,"version":"Nemotron Nano v2"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":146,"score":61.82,"version":"Granite 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":147,"score":61.4,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":148,"score":61.39,"version":"Granite 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":149,"score":61.3,"version":"Qwen2.5 Omni"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":150,"score":61.1,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":151,"score":60.3,"version":"Llama 3.1"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":152,"score":59.6,"version":"Marin 32B"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":153,"score":59.01,"version":"Granite 3.3"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":154,"score":58.1,"version":"HY"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":155,"score":58.09,"version":"Falcon 3"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":156,"score":57.9,"version":"Dream 7B"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":157,"score":57.84,"version":"Trinity Large"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":158,"score":56.2,"version":"Ministral"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":159,"score":56.12,"version":"Apertus"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":160,"score":56.1,"version":"OLMo 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":161,"score":55.71,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":162,"score":55.37,"version":"Qwen2.5"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":163,"score":55.33,"version":"EuroLLM"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":164,"score":55.2,"version":"Marin 32B"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":165,"score":54.43,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":166,"score":53.76,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":167,"score":53.72,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":168,"score":53.69,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":169,"score":53.51,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":170,"score":52.51,"version":"Granite 3.3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":171,"score":52.12,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":172,"score":51.07,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":173,"score":51.06,"version":"Llama 3.1"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":174,"score":50.1,"version":"EXAONE 3.5"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":175,"score":49.54,"version":"LFM2.5-Audio"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":176,"score":48.58,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":177,"score":47.47,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":178,"score":47.4,"version":"LLaDA"},{"company":"kyutai","family":"Moshi","model":"moshi","place":179,"score":47.4,"version":"Moshi"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":180,"score":46.55,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":181,"score":46.47,"version":"Falcon H1"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":182,"score":46.4,"version":"Mini-Omni2"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b-instruct","place":183,"score":46.4,"version":"EXAONE 3.5"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":184,"score":45.02,"version":"Falcon 3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-mini-instruct","place":185,"score":44.5,"version":"Mi:dm 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":186,"score":43.66,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":187,"score":42.91,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":188,"score":42.41,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":189,"score":41.47,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":190,"score":39.45,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":191,"score":39.1,"version":"Gemma 3"},{"company":"loka","family":"TRLM","model":"trlm-135m","place":192,"score":36.8,"version":"TRLM"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":193,"score":35.3,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":194,"score":35.18,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":195,"score":34.47,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":196,"score":33.26,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":197,"score":33.21,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":198,"score":32.95,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":199,"score":32.2,"version":"SmolLM2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":200,"score":31,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":201,"score":30.72,"version":"Llama 3.2"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":202,"score":30.54,"version":"LFM2-Audio"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":203,"score":30.26,"version":"Gemma 3"},{"company":"arcee","family":"Llama Spark","model":"llama-spark-8b","place":204,"score":29.77,"version":"Llama Spark 8B"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":205,"score":28.2,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-1b","place":206,"score":27.6,"version":"Llama 1"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":207,"score":26.7,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":208,"score":25.7,"version":"SmolLM1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":209,"score":21.7,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":210,"score":12.64,"version":"SmolLM2"},{"company":"tii","family":"Falcon","model":"falcon-e-1b-base","place":211,"score":12.28,"version":"Falcon E"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":212,"score":11.75,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":213,"score":6.76,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":214,"score":5.54,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":215,"score":2.5,"version":"Gemma 2"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":216,"score":0.9,"version":"Magpie"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":217,"score":0.706,"version":"Llama 3.3"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":218,"score":0.7,"version":"Llama 3.1 Nemotron"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":219,"score":0.652,"version":"Llama TFree HAT"}],"stars":"510","top_company":"baidu","top_model":"ERNIE 4.5 424B A47B","top_model_family":"ERNIE","top_model_id":"ernie-4.5","top_model_version":"ERNIE 4.5","top_open_source":1,"top_percent":94.3,"topic":"Multi-task reasoning","total_models":1613,"url":"https://github.com/suzgunmirac/BIG-bench-Hard"},{"bench_models":10,"description":"Chinese translation of BIG-Bench Hard reasoning tasks.","github":"","has_recent":0,"id":"bbh-zh","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BBH-ZH","paper":"","placements":[{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":1,"score":87.52,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":2,"score":86.23,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":3,"score":86.18,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":4,"score":82.55,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":5,"score":80.09,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":6,"score":75.75,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":7,"score":75.09,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":8,"score":70.4,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":9,"score":67.65,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":10,"score":66.11,"version":"Ling 2.0"}],"stars":"","top_company":"antgroup","top_model":"LLaDA2.0 Flash","top_model_family":"LLaDA","top_model_id":"llada2.0-flash","top_model_version":"LLaDA 2.0","top_open_source":1,"top_percent":87.52,"topic":"Multi-task reasoning (Chinese)","total_models":1613,"url":""},{"bench_models":12,"description":"Bias Benchmark for Question Answering evaluating social biases across contexts.","github":"https://github.com/jinjh0123/BBG","has_recent":0,"id":"bbq","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BBQ","paper":"https://arxiv.org/abs/2110.08193","placements":[{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":1,"score":55.98,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":2,"score":51.5,"version":"Llama 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":3,"score":0.6866,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":4,"score":0.6405,"version":"Claude 3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":5,"score":0.6362,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo","place":6,"score":0.6362,"version":"GPT-3.5"},{"company":"hcx","family":"HCX","model":"hcx","place":7,"score":0.6345,"version":"HCX"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":8,"score":0.6228,"version":"Llama 3.3"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":9,"score":0.6026,"version":"Gemini 2.0"},{"company":"hcx","family":"HCX","model":"hcx-dash","place":10,"score":0.5966,"version":"HCX"},{"company":"openai","family":"GPT","model":"gpt-4o","place":11,"score":0.5733,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":12,"score":0.547,"version":"Claude 3.5"}],"stars":"","top_company":"mistral","top_model":"Mixtral 8x 7B","top_model_family":"Mixtral","top_model_id":"mixtral-8x7b","top_model_version":"Mixtral","top_open_source":1,"top_percent":55.98,"topic":"Bias evaluation","total_models":1613,"url":"https://huggingface.co/datasets/heegyu/bbq"},{"bench_models":4,"description":"Safety benchmark evaluating harmfulness in model responses.","github":"","has_recent":0,"id":"beavertails","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BeaverTails","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":1,"score":76.7,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":2,"score":70.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":3,"score":68,"version":"Qwen2.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":4,"score":67.5,"version":"IQuest Coder V1"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Thinking","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-thinking","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":76.7,"topic":"Safety / harmfulness","total_models":1613,"url":""},{"bench_models":13,"description":"Advanced math problems exceeding AIME difficulty.","github":"","has_recent":0,"id":"beyondaime","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BeyondAIME","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":1,"score":86.5,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":86,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":83,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":82,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":5,"score":77,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":6,"score":76,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":74,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":8,"score":72,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":69,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":10,"score":69,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":62,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":62,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-moe-2507","place":13,"score":57.6,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":57,"version":"Claude 4.5"}],"stars":"","top_company":"bytedance","top_model":"Seed2.0 Pro","top_model_family":"Seed","top_model_id":"seed2.0-pro","top_model_version":"Seed 2.0","top_open_source":0,"top_percent":86.5,"topic":"Math (beyond AIME)","total_models":1613,"url":""},{"bench_models":17,"description":"Benchmark for functional code correctness and logic.","github":"","has_recent":0,"id":"bfcl","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BFCL","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":1,"score":95,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":2,"score":92.3,"version":"Llama 3.1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":3,"score":92.3,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":4,"score":89.5,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":5,"score":88.5,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":6,"score":84.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":7,"score":83,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":8,"score":76.1,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":9,"score":70.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":10,"score":70.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":11,"score":69.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":12,"score":63.4,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":13,"score":63,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":14,"score":61.5,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":15,"score":60.3,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":16,"score":58.6,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":17,"score":55.34,"version":"Motif 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-4B","top_model_family":"Qwen","top_model_id":"qwen3-4b","top_model_version":"Qwen3","top_open_source":1,"top_percent":95,"topic":"Code reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"Financial compliance and literacy questions from the BFCL Live v2 benchmark.","github":"","has_recent":0,"id":"bfcl-live-v2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BFCL Live v2","paper":"","placements":[{"company":"openai","family":"o1","model":"o1-mini","place":1,"score":81,"version":"o1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":2,"score":79,"version":"QwQ"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":3,"score":75.43,"version":"LLaDA 2.0"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":4,"score":75.4,"version":"Apriel Nemotron"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":5,"score":75.4,"version":"EXAONE 4.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":6,"score":74.11,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":7,"score":73.19,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":8,"score":67.57,"version":"Ling 2.0"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":9,"score":63.9,"version":"Llama 3.1"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":10,"score":54.1,"version":"Llama 3.1"}],"stars":"","top_company":"openai","top_model":"o1 Mini","top_model_family":"o1","top_model_id":"o1-mini","top_model_version":"o1","top_open_source":0,"top_percent":81,"topic":"Finance QA","total_models":1613,"url":""},{"bench_models":2,"description":"Second release of the BFCL benchmark focusing on functional code correctness and logic.","github":"","has_recent":0,"id":"bfcl-v2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BFCL v2","paper":"","placements":[{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":1,"score":29.4,"version":"MobileLLM"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":2,"score":25.7,"version":"Llama 3.2"}],"stars":"","top_company":"meta","top_model":"MobileLLM P1","top_model_family":"MobileLLM","top_model_id":"mobilellm-p1","top_model_version":"MobileLLM","top_open_source":1,"top_percent":29.4,"topic":"Code reasoning","total_models":1613,"url":""},{"bench_models":81,"description":"Benchmark for functional code correctness and logic (v3).","github":"","has_recent":0,"id":"bfcl-v3","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BFCL v3","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.5","place":1,"score":77.8,"version":"GLM 4.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":2,"score":76.5,"version":"openPangu-R"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":3,"score":75.61,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":4,"score":74.94,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":5,"score":74.86,"version":"LLaDA 2.1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":6,"score":74.6,"version":"openPangu-R"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":7,"score":73.61,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":8,"score":73.41,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":9,"score":72.4,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":10,"score":72.06,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":11,"score":72,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":12,"score":71.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":13,"score":70.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":14,"score":70.8,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":15,"score":70.72,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":16,"score":70.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":17,"score":70.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":18,"score":70.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":19,"score":70.3,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":20,"score":70.12,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":21,"score":69.1,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":22,"score":68.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":23,"score":68.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":24,"score":68.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":25,"score":68,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":26,"score":68,"version":"TeleChat3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":27,"score":67.9,"version":"DeepSeek R1"},{"company":"openai","family":"o1","model":"o1","place":28,"score":67.8,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":29,"score":67.8,"version":"o1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":30,"score":67.69,"version":"Ling 2.0"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":31,"score":67.5,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":32,"score":67,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":33,"score":66.9,"version":"Nemotron Nano v2"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":34,"score":66.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":35,"score":66.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":36,"score":66.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":37,"score":66.3,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":38,"score":65.9,"version":"TeleChat3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":39,"score":65.25,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":40,"score":65.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":41,"score":65.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":42,"score":65,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":43,"score":64.69,"version":"Granite 4.0"},{"company":"openai","family":"o3","model":"o3-mini","place":44,"score":64.6,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":45,"score":64.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":46,"score":64.4,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":47,"score":64.4,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":48,"score":63.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":49,"score":63.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":50,"score":63,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":51,"score":62.9,"version":"Gemini 2.5"},{"company":"essential","family":"Rnj","model":"rnj-1","place":52,"score":62.2,"version":"Rnj 1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":53,"score":61.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":54,"score":61.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":55,"score":60.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":56,"score":60.2,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":57,"score":60.2,"version":"Motif 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":58,"score":59.98,"version":"Granite 4.0"},{"company":"arcee","family":"Trinity","model":"trinity-mini","place":59,"score":59.67,"version":"Trinity"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":60,"score":58.21,"version":"Granite 4.0"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":61,"score":58,"version":"Youtu-LLM"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":62,"score":56.97,"version":"LFM2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":63,"score":56.9,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":64,"score":56.6,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":65,"score":55.93,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":66,"score":55.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":67,"score":55.41,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":68,"score":53.75,"version":"Ling 2.0"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":69,"score":53.61,"version":"GPT-OSS"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":70,"score":52.9,"version":"EXAONE 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":71,"score":52.43,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":72,"score":52.2,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":73,"score":50.69,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":74,"score":50.6,"version":"Gemma 3"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":75,"score":50.5,"version":"Codestral"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":76,"score":49.12,"version":"LFM2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":77,"score":48.9,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":78,"score":46.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":79,"score":46.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":80,"score":45.9,"version":"GPT-OSS"},{"company":"mistral","family":"Magistral","model":"magistral-small-2509","place":81,"score":44.25,"version":"Magistral 1.2"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":82,"score":37.1,"version":"SmolLM3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":83,"score":31.5,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":84,"score":25.9,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":85,"score":21.44,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":86,"score":21.44,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":87,"score":16.64,"version":"Gemma 3"}],"stars":"","top_company":"zai","top_model":"GLM 4.5","top_model_family":"GLM","top_model_id":"glm-4.5","top_model_version":"GLM 4.5","top_open_source":1,"top_percent":77.8,"topic":"Code reasoning","total_models":1613,"url":""},{"bench_models":8,"description":"BFCL v3 Live subset for real-time tool calling evaluation.","github":"","has_recent":0,"id":"bfcl-v3-live","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BFCL v3 (Live)","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":1,"score":82.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":2,"score":80.3,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":3,"score":76.66,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":4,"score":75.9,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":5,"score":75.6,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":6,"score":74.3,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":7,"score":73.93,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":8,"score":69.14,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":9,"score":68.74,"version":"Kanana 1.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":82.9,"topic":"Tool calling","total_models":1613,"url":""},{"bench_models":13,"description":"BFCL v3 Multi-Turn subset for multi-turn tool calling evaluation.","github":"","has_recent":0,"id":"bfcl-v3-multi-turn","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BFCL v3 (Multi-Turn)","paper":"","placements":[{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":1,"score":76.8,"version":"MiniMax M2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":68,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":3,"score":63.3,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":61,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":5,"score":53.6,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":6,"score":43.7,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":7,"score":38.77,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":8,"score":38.63,"version":"Kanana 2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":9,"score":37.4,"version":"MiniMax M2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":10,"score":35.6,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":11,"score":35.38,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":12,"score":34.3,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":13,"score":11.88,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":14,"score":11.38,"version":"Kanana 1.5"}],"stars":"","top_company":"minimax","top_model":"MiniMax M2.5","top_model_family":"MiniMax","top_model_id":"minimax-m2.5","top_model_version":"MiniMax M2.5","top_open_source":1,"top_percent":76.8,"topic":"Tool calling","total_models":1613,"url":""},{"bench_models":28,"description":"BFCL v4 benchmark for functional code correctness and logic.","github":"","has_recent":0,"id":"bfcl-v4","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BFCL v4","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":77.5,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":2,"score":72.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":72.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":72.2,"version":"Qwen3.5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":5,"score":69.87,"version":"Ling 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":68.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":7,"score":68.5,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":8,"score":68.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":9,"score":67.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":10,"score":67.3,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":11,"score":63.1,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":12,"score":63.1,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":13,"score":63.05,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":14,"score":62.96,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":62.5,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":16,"score":61.6,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":17,"score":61.2,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":18,"score":60.05,"version":"DeepSeek V3.2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":19,"score":57.2,"version":"Seed 1.8"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":20,"score":56.5,"version":"Nanbeige4.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":21,"score":55.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":22,"score":54.8,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":23,"score":53.8,"version":"Nanbeige4"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":24,"score":53.8,"version":"Nemotron 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":25,"score":52.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":26,"score":48.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":27,"score":48.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":28,"score":47.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":29,"score":46.4,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":30,"score":45.99,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":31,"score":45.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":32,"score":45.14,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":33,"score":44.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":34,"score":44.87,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":35,"score":42.2,"version":"Qwen3"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":77.5,"topic":"Code reasoning","total_models":1613,"url":""},{"bench_models":2,"description":"BIG-bench overall performance (original).","github":"https://github.com/google/BIG-bench","has_recent":0,"id":"big-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BIG-Bench","paper":"","placements":[{"company":"google","family":"Gemma","model":"gemma-2-7b","place":1,"score":55.1,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":2,"score":35.2,"version":"Gemma 2"}],"stars":"3110","top_company":"google","top_model":"Gemma 2 7B","top_model_family":"Gemma","top_model_id":"gemma-2-7b","top_model_version":"Gemma 2","top_open_source":1,"top_percent":55.1,"topic":"Multi-task reasoning","total_models":1613,"url":"https://github.com/google/BIG-bench"},{"bench_models":23,"description":"Extra hard subset of BIG-bench tasks.","github":"","has_recent":0,"id":"big-bench-extra-hard","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BIG-Bench Extra Hard","paper":"","placements":[{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":1,"score":51.99,"version":"Ling 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":2,"score":48.43,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":3,"score":48.04,"version":"DeepSeek V3.2"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":4,"score":47.34,"version":"Ling 1T"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":5,"score":47.25,"version":"Ling 1T"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":43.12,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5","place":7,"score":42.86,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":8,"score":39.75,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":9,"score":37.8,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":10,"score":35.77,"version":"LLaDA 2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":11,"score":34.83,"version":"DeepSeek V3.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":12,"score":33.51,"version":"LLaDA 2.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":29.08,"version":"Gemini 2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":14,"score":27.86,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":15,"score":23.24,"version":"Ling 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":16,"score":19.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":17,"score":18.27,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":18,"score":16.47,"version":"LLaDA 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":19,"score":16.3,"version":"Gemma 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":20,"score":15.78,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":21,"score":15.3,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":22,"score":14.81,"version":"Ling 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":23,"score":11,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":24,"score":7.2,"version":"Gemma 3"}],"stars":"","top_company":"antgroup","top_model":"Ling 2.5 1T","top_model_family":"Ling","top_model_id":"ling-2.5-1t","top_model_version":"Ling 2.5","top_open_source":1,"top_percent":51.99,"topic":"Multi-task reasoning","total_models":1613,"url":""},{"bench_models":157,"description":"BigCodeBench evaluates large language models on practical code generation tasks with unit-test verification.","github":"https://github.com/bigcode-project/BigCodeBench","has_recent":0,"id":"bigcodebench","leaderboard":"https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard","lower_is_better":false,"metric":"pass@1","name":"BigCodeBench","paper":"https://arxiv.org/abs/2409.00001","placements":[{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":1,"score":70.1,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":2,"score":63,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":3,"score":62.9,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":4,"score":61.7,"version":"Kimi K2"},{"company":"essential","family":"Rnj","model":"rnj-1","place":5,"score":57.1,"version":"Rnj 1"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-05-13","place":6,"score":56.1,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":7,"score":56.1,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":8,"score":55.7,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":9,"score":55.5,"version":"Llama 4"},{"company":"quasar","family":"Quasar","model":"quasar-alpha","place":10,"score":55.1,"version":"Quasar"},{"company":"google","family":"Gemini","model":"gemini-exp-1206","place":11,"score":54.7,"version":"Gemini 1.0"},{"company":"google","family":"Gemini","model":"gemini-exp-1114","place":12,"score":54.3,"version":"Gemini 1.0"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":13,"score":54.2,"version":"IQuest Coder V1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2-chat","place":14,"score":54.1,"version":"DeepSeek Coder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":15,"score":54,"version":"DeepSeek Coder V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":16,"score":53.5,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":17,"score":53.5,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":18,"score":53.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":19,"score":53.2,"version":"GPT-4"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-exp","place":20,"score":52.9,"version":"Gemini 2.0"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20240620","place":21,"score":52.7,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku-20241022","place":22,"score":52.5,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":23,"score":52.5,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":24,"score":52.2,"version":"Llama 3.3"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":25,"score":51.8,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-exp-1121","place":26,"score":51.8,"version":"Gemini 1.0"},{"company":"openai","family":"GPT","model":"gpt-4-0613","place":27,"score":51.6,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-3-opus-20240229","place":28,"score":51.5,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":29,"score":51.4,"version":"Claude 4.5"},{"company":"nvidia","family":"Athene","model":"athene-v2-agent","place":30,"score":51.2,"version":"Athene V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5-1210","place":31,"score":50.9,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":32,"score":50.8,"version":"Qwen2.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-api-0514","place":33,"score":50.6,"version":"Gemini 1.5"},{"company":"nousresearch","family":"Hermes","model":"hermes-2-theta-llama-3-70b","place":34,"score":50.6,"version":"Hermes 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":35,"score":50.5,"version":"Llama 3.1"},{"company":"microsoft","family":"Phi","model":"phi-4","place":36,"score":50.5,"version":"Phi 3.5"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":37,"score":50.5,"version":"Codestral"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":38,"score":50.2,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":39,"score":49.9,"version":"IQuest Coder V1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":40,"score":49.8,"version":"Kimi K2"},{"company":"qwen","family":"QwQ","model":"qwq-32b-preview","place":41,"score":49.5,"version":"QwQ"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":42,"score":49.4,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":43,"score":49.4,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-api-0514","place":44,"score":49.3,"version":"Gemini 1.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":45,"score":48.6,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3-70b-synthia-v3.5","place":46,"score":48.5,"version":"Llama 1"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet-20240229","place":47,"score":48.3,"version":"Claude 3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":48,"score":48.3,"version":"KAT"},{"company":"novasky","family":"Sky","model":"sky-t1-32b-flash","place":49,"score":48.2,"version":"Sky T1"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":50,"score":48.2,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":51,"score":48.1,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":52,"score":48,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-24b-instruct-2501","place":53,"score":47.8,"version":"Mistral"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-ds-33b","place":54,"score":47.6,"version":"Reflectioncoder"},{"company":"google","family":"Gemma","model":"gemma-2-27b-instruct","place":55,"score":47.6,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":56,"score":47.4,"version":"Qwen3"},{"company":"whiterabbit","family":"WhiteRabbitNeo-33B-v1","model":"whiterabbitneo-33b-v1","place":57,"score":47.2,"version":"WhiteRabbitNeo-33B-v1"},{"company":"mistral","family":"Codestral","model":"codestral-22b-v0.1","place":58,"score":47.1,"version":"Codestral v0.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":59,"score":47.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":60,"score":47,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":61,"score":46.9,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":62,"score":46.8,"version":"Kimi K2"},{"company":"autocoder","family":"Autocoder","model":"autocoder","place":63,"score":46.6,"version":"Autocoder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":64,"score":46.5,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen2-72b-chat","place":65,"score":46.3,"version":"Qwen2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":66,"score":46.23,"version":"Granite 4.0"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":67,"score":46.2,"version":"KAT"},{"company":"nousresearch","family":"Hermes","model":"hermes-2-pro-llama-3-70b","place":68,"score":45.9,"version":"Hermes 3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":69,"score":45.7,"version":"GLM 4.7"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b-instruct","place":70,"score":45.4,"version":"Mixtral"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":71,"score":45.4,"version":"Kimi Dev"},{"company":"codellama","family":"CodeLlama","model":"codellama-70b-instruct","place":72,"score":45.2,"version":"CodeLlama"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0125","place":73,"score":44.9,"version":"GPT-3.5"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku-20240307","place":74,"score":44.8,"version":"Claude 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2-chat","place":75,"score":44.7,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":76,"score":44.6,"version":"Qwen2.5"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":77,"score":44.6,"version":"Seed Coder"},{"company":"zhongguancun","family":"Codegeex4","model":"codegeex4-all-9b","place":78,"score":44.5,"version":"Codegeex4"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":79,"score":44.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":80,"score":44.1,"version":"Qwen3"},{"company":"deepcoder","family":"Deepcoder","model":"deepcoder-14b-preview","place":81,"score":43.9,"version":"Deepcoder"},{"company":"01ai","family":"Yi 1.5","model":"yi-coder-9b-chat","place":82,"score":43.5,"version":"Yi 1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-nemotron-70b-instruct","place":83,"score":43.5,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-14b","place":84,"score":43.3,"version":"DeepSeek R1"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-128k-instruct","place":85,"score":43.2,"version":"Phi 3.5"},{"company":"phind","family":"Phind-CodeLlama-34B-v2","model":"phind-codellama-34b-v2","place":86,"score":42.9,"version":"Phind-CodeLlama-34B-v2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-70b","place":87,"score":42.6,"version":"DeepSeek R1"},{"company":"artigenz","family":"Artigenz-Coder-DS-6.7B","model":"artigenz-coder-ds-6.7b","place":88,"score":42.5,"version":"Artigenz Coder DS"},{"company":"01ai","family":"Yi 1.5","model":"yi-large","place":89,"score":42.5,"version":"Yi 1.5"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-ds-6.7b","place":90,"score":42.3,"version":"Reflectioncoder"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-cl-34b","place":91,"score":42.3,"version":"Reflectioncoder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":92,"score":42.2,"version":"DeepSeek Coder V2"},{"company":"nvidia","family":"Athene","model":"athene-70b","place":93,"score":42.1,"version":"Athene V2"},{"company":"nexusflow","family":"Nxcode","model":"nxcode-cq-7b-orpo","place":94,"score":42.1,"version":"Nxcode"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":95,"score":41.9,"version":"Magicoder-S-DS-6.7B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":96,"score":41.9,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b-chat","place":97,"score":41.6,"version":"CodeQwen1.5"},{"company":"tess","family":"Tess","model":"tess-v2.5.2-qwen2-72b","place":98,"score":41.6,"version":"Tess"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":99,"score":41.58,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":100,"score":41.58,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.57b-a14b","place":101,"score":41.5,"version":"Qwen2"},{"company":"microsoft","family":"Phi","model":"phi-3.1-mini-128k-instruct","place":102,"score":41.5,"version":"Phi 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":103,"score":41.49,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":104,"score":41.49,"version":"Qwen3"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b-instruct-v0.1","place":105,"score":41.4,"version":"StarCoder2"},{"company":"mistral","family":"Mistral","model":"mistral-small-instruct-2409","place":106,"score":41.4,"version":"Mistral"},{"company":"autocoder","family":"Autocoder","model":"autocoder-s-6.7b","place":107,"score":41.3,"version":"Autocoder"},{"company":"cohere","family":"C4ai","model":"c4ai-command-r-08-2024","place":108,"score":41.2,"version":"C4ai"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":109,"score":41,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":110,"score":40.92,"version":"Granite 4.0"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-6.7b","place":111,"score":40.9,"version":"OpenCodeInterpreter DS"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":112,"score":40.84,"version":"Granite 4.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":113,"score":40.7,"version":"Ling 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":114,"score":40.7,"version":"Ling 2.0"},{"company":"IBM","family":"Granite","model":"granite-code-34b-instruct","place":115,"score":40.3,"version":"Granite 3.1"},{"company":"mistral","family":"Codestral","model":"codestral-mamba","place":116,"score":40.3,"version":"Codestral"},{"company":"qwen","family":"Qwen","model":"qwen1.5-110b-chat","place":117,"score":39.7,"version":"Qwen1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":118,"score":39.6,"version":"DeepSeek Coder"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":119,"score":39.21,"version":"Granite 4.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":120,"score":39.21,"version":"LLaDA 2.1"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-34b-chat","place":121,"score":38.8,"version":"Yi 1.5"},{"company":"wavecoder","family":"WaveCoder-Ultra-6.7B","model":"wavecoder-ultra-6.7b","place":122,"score":38.8,"version":"WaveCoder-Ultra-6.7B"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-9b-chat","place":123,"score":38.5,"version":"Yi 1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":124,"score":38.3,"version":"Llama 3.1"},{"company":"cohere","family":"Command","model":"command-r-plus","place":125,"score":37.8,"version":"Command"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":126,"score":37.8,"version":"DeepSeek Coder V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":127,"score":37.8,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":128,"score":37.7,"version":"Gemma 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":129,"score":37.11,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":130,"score":37.1,"version":"Qwen1.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-72b-chat","place":131,"score":36.8,"version":"Qwen1.5"},{"company":"map","family":"OpenCoder","model":"opencoder-1-5b-instruct","place":132,"score":36.8,"version":"OpenCoder-8B-Instruct"},{"company":"mistral","family":"Mistral","model":"mistral-small-2402","place":133,"score":36.7,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":134,"score":36.6,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":135,"score":36.05,"version":"Qwen3"},{"company":"codegemma","family":"codegemma-2b","model":"codegemma-7b-instruct","place":136,"score":35.8,"version":"Codegemma"},{"company":"qwen","family":"Qwen","model":"qwen2-7b-instruct","place":137,"score":35.6,"version":"Qwen2"},{"company":"microsoft","family":"Phi","model":"phi-3.5-mini-instruct","place":138,"score":35.6,"version":"Phi 3.5"},{"company":"microsoft","family":"Phi","model":"phi-3-small-128k-instruct","place":139,"score":35.3,"version":"Phi 3.5"},{"company":"microsoft","family":"Phi","model":"phi-3-mini-128k-instruct","place":140,"score":35.1,"version":"Phi 3.5"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":141,"score":35,"version":"Ling 2.0"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-cl-7b","place":142,"score":34.5,"version":"Reflectioncoder"},{"company":"meta","family":"Llama3-8B-instruct","model":"llama3-8b-instruct","place":143,"score":34.4,"version":"Llama3-8B-instruct"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":144,"score":34.39,"version":"LLaDA 2.1"},{"company":"mistral","family":"Mistral","model":"mistral-large-2402","place":145,"score":34.1,"version":"Mistral"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":146,"score":33.4,"version":"OLMo 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":147,"score":32.89,"version":"LLaDA 2.0"},{"company":"openchat","family":"OpenChat-3.5-7B-0106","model":"openchat-3-6-8b-20240522","place":148,"score":32.6,"version":"OpenChat-3.5-7B-0106"},{"company":"IBM","family":"Granite","model":"granite-3.0-8b-instruct","place":149,"score":32.4,"version":"Granite 3.1"},{"company":"meta","family":"CodeLlama","model":"codellama-34b-instruct","place":150,"score":32.3,"version":"CodeLlama"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":151,"score":30.44,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":152,"score":30.18,"version":"LLaDA 2.1"},{"company":"meta","family":"CodeLlama","model":"codellama-13b-instruct","place":153,"score":30.1,"version":"CodeLlama"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-1.5b-instruct","place":154,"score":29.9,"version":"Qwen2.5"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-6b-chat","place":155,"score":29.8,"version":"Yi 1.5"},{"company":"internlm","family":"Intern","model":"internlm2.5-7b-chat","place":156,"score":29.5,"version":"Internlm2.5"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-1.3b","place":157,"score":27.6,"version":"OpenCodeInterpreter DS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-instruct","place":158,"score":26.2,"version":"DeepSeek Coder"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":159,"score":25.9,"version":"Llama 3.2"},{"company":"meta","family":"CodeLlama","model":"codellama-7b-instruct","place":160,"score":23.8,"version":"CodeLlama"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1-5b-instruct","place":161,"score":23.7,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.3","place":162,"score":23.4,"version":"Mistral 0.3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":163,"score":21.2,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":164,"score":12.9,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b-instruct","place":165,"score":10.1,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":166,"score":9.8,"version":"Llama 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":167,"score":7.5,"version":"DeepSeek R1"}],"stars":"","top_company":"xiaomi","top_model":"MiMo V2 Flash Base","top_model_family":"MiMo","top_model_id":"mimo-v2-flash-base","top_model_version":"MiMo V2","top_open_source":1,"top_percent":70.1,"topic":"Code Generation","total_models":1613,"url":"https://huggingface.co/datasets/bigcode/bigcodebench"},{"bench_models":168,"description":"Harder variant of BigCodeBench testing complex programming and library tasks with function-level code generation.","github":"https://github.com/bigcode-project/bigcodebench","has_recent":0,"id":"bigcodebench-hard","leaderboard":"https://bigcode-project.github.io/bigcodebench/","lower_is_better":false,"metric":"pass@1","name":"BigCodeBench Hard","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":1,"score":35.8,"version":"Claude 3.7"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":2,"score":35.5,"version":"o1"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":3,"score":35.5,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":4,"score":35.1,"version":"DeepSeek R1"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":5,"score":35.1,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":35.1,"version":"Claude 4.5"},{"company":"quasar","family":"Quasar","model":"quasar-alpha","place":7,"score":34.8,"version":"Quasar"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":8,"score":34.5,"version":"o1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":9,"score":34.5,"version":"DeepSeek V3"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":10,"score":34.5,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-exp-1206","place":11,"score":34.1,"version":"Gemini 1.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-exp-03.25","place":12,"score":33.1,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":13,"score":33.1,"version":"Grok 3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":14,"score":33.1,"version":"IQuest Coder V1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":15,"score":32.8,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":16,"score":32.8,"version":"Claude 3.7"},{"company":"openbmb","family":"Optimus","model":"optimus-alpha","place":17,"score":32.8,"version":"Optimus"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":18,"score":32.8,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-max","place":19,"score":32.5,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":20,"score":32.1,"version":"GPT-4"},{"company":"nvidia","family":"Athene","model":"athene-v2-chat","place":21,"score":32.1,"version":"Athene V2"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro-exp-02-05","place":22,"score":31.8,"version":"Gemini 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":23,"score":31.8,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini-2025-04-14","place":24,"score":31.8,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":25,"score":31.8,"version":"Kimi Dev"},{"company":"nvidia","family":"Athene","model":"athene-v2-agent","place":26,"score":31.4,"version":"Athene V2"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":27,"score":31.1,"version":"GPT-4o"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta","place":28,"score":31.1,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":29,"score":30.8,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":30,"score":30.8,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":31,"score":30.4,"version":"Claude 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":32,"score":30.4,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku-20241022","place":33,"score":30.1,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano-2025-04-14","place":34,"score":30.1,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20240620","place":35,"score":29.4,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":36,"score":29.4,"version":"DeepSeek Coder V2"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-exp-0827","place":37,"score":29.4,"version":"Gemini 1.0"},{"company":"google","family":"Gemini","model":"gemini-exp-1114","place":38,"score":29.4,"version":"Gemini 1.0"},{"company":"openai","family":"GPT","model":"chatgpt-4o-latest-20250129","place":39,"score":29.4,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":40,"score":29.1,"version":"Claude 4.5"},{"company":"openai","family":"o1","model":"o1-preview-2024-09-12","place":41,"score":28.8,"version":"o1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2-chat","place":42,"score":28.7,"version":"DeepSeek Coder"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":43,"score":28.7,"version":"Gemini 2.0"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":44,"score":28.4,"version":"Llama 3.3"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-exp","place":45,"score":28.4,"version":"Gemini 2.0"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":46,"score":28.4,"version":"Llama 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":47,"score":28.4,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":48,"score":27.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":49,"score":27.7,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":50,"score":27.7,"version":"IQuest Coder V1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-exp-0801","place":51,"score":27.4,"version":"Gemini 1.0"},{"company":"openai","family":"o1","model":"o1-mini-2024-09-12","place":52,"score":27.4,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-exp-1121","place":53,"score":27.4,"version":"Gemini 1.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-thinking-exp-1219","place":54,"score":27.4,"version":"Gemini 2.0"},{"company":"microsoft","family":"Phi","model":"phi-4","place":55,"score":27.4,"version":"Phi 3.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-05-13","place":56,"score":27.1,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":57,"score":27,"version":"DeepSeek V3.2"},{"company":"novasky","family":"Sky","model":"sky-t1-32b-flash","place":58,"score":26.7,"version":"Sky T1"},{"company":"qwen","family":"QwQ","model":"qwq-32b-preview","place":59,"score":26.7,"version":"QwQ"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":60,"score":26.6,"version":"Gemini 1.5"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":61,"score":26.6,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":62,"score":26.6,"version":"Qwen2.5"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta","place":63,"score":26.6,"version":"Grok 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":64,"score":26.4,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5.1210","place":65,"score":26.4,"version":"DeepSeek Coder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":66,"score":26.4,"version":"DeepSeek R1"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":67,"score":26.4,"version":"KAT"},{"company":"zai","family":"GLM","model":"glm-4.7","place":68,"score":26.4,"version":"GLM 4.7"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":69,"score":26.1,"version":"DeepSeek V2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":70,"score":25.7,"version":"Qwen3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":71,"score":25.7,"version":"KAT"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":72,"score":25.6,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-api-0514","place":73,"score":25.4,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":74,"score":25.4,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":75,"score":25.4,"version":"Qwen2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-lite-preview-02-05","place":76,"score":25.4,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":77,"score":25.3,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":78,"score":25,"version":"Gemini 3"},{"company":"meta","family":"Llama","model":"llama-3.70b-instruct","place":79,"score":24.6,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":80,"score":24.6,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-nemotron-70b-instruct","place":81,"score":24.6,"version":"Llama 3.1"},{"company":"dracarys","family":"Dracarys","model":"dracarys-llama-3.1-70b-instruct","place":82,"score":24.3,"version":"Dracarys Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":83,"score":24.3,"version":"Qwen2.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-api-0514","place":84,"score":23.6,"version":"Gemini 1.0"},{"company":"meta","family":"Llama","model":"llama-3.70b-synthia-v3.5","place":85,"score":23.6,"version":"Llama 1"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":86,"score":23.6,"version":"Seed Coder"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet-20240229","place":87,"score":23.4,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":88,"score":23,"version":"Qwen3"},{"company":"dracarys","family":"Dracarys","model":"dracarys-72b-instruct","place":89,"score":22.6,"version":"Dracarys"},{"company":"nousresearch","family":"Hermes","model":"hermes-2-theta-llama-3.70b","place":90,"score":22.3,"version":"Hermes 3"},{"company":"nousresearch","family":"Hermes","model":"hermes-2-pro-llama-3.70b","place":91,"score":21.6,"version":"Hermes 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-exp-0827","place":92,"score":21.6,"version":"Gemini 1.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-14b-instruct","place":93,"score":20.9,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-14b","place":94,"score":20.9,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen2-72b-chat","place":95,"score":20.6,"version":"Qwen2"},{"company":"mistral","family":"Codestral","model":"codestral-22b-v0.1","place":96,"score":20.6,"version":"Codestral v0.1"},{"company":"mistral","family":"Mistral","model":"mistral-small-24b-instruct-2501","place":97,"score":20.6,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":98,"score":20.3,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-70b","place":99,"score":20.3,"version":"DeepSeek R1"},{"company":"deepcoder","family":"Deepcoder","model":"deepcoder-14b-preview","place":100,"score":20.3,"version":"Deepcoder"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0125","place":101,"score":19.9,"version":"GPT-3.5"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b-instruct","place":102,"score":19.9,"version":"Mixtral"},{"company":"nvidia","family":"Athene","model":"athene-70b","place":103,"score":19.9,"version":"Athene V2"},{"company":"deepcoder","family":"Deepcoder","model":"deepcoder-14b-preview","place":104,"score":19.6,"version":"Deepcoder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":105,"score":19.3,"version":"DeepSeek Coder"},{"company":"whiterabbit","family":"WhiteRabbitNeo-33B-v1","model":"whiterabbitneo-33b-v1","place":106,"score":19.3,"version":"WhiteRabbitNeo-33B-v1"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-ds-33b","place":107,"score":18.9,"version":"Reflectioncoder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":108,"score":18.9,"version":"DeepSeek Coder V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2-chat","place":109,"score":18.6,"version":"DeepSeek Coder"},{"company":"map","family":"OpenCoder","model":"opencoder-8b-instruct","place":110,"score":18.5,"version":"OpenCoder"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku-20240307","place":111,"score":18.3,"version":"Claude 3"},{"company":"openai","family":"GPT","model":"gpt-4.0613","place":112,"score":17.6,"version":"GPT-4"},{"company":"01ai","family":"Yi 1.5","model":"yi-large","place":113,"score":17.6,"version":"Yi 1.5"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b-chat","place":114,"score":17.2,"version":"CodeQwen1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":115,"score":17.2,"version":"DeepSeek Coder V2"},{"company":"zhongguancun","family":"Codegeex4","model":"codegeex4-all-9b","place":116,"score":17.2,"version":"Codegeex4"},{"company":"nexusflow","family":"Nxcode","model":"nxcode-cq-7b-orpo","place":117,"score":17.2,"version":"Nxcode"},{"company":"autocoder","family":"Autocoder","model":"autocoder","place":118,"score":16.9,"version":"Autocoder"},{"company":"meta","family":"Llama","model":"llama-4-scout","place":119,"score":16.9,"version":"Llama 1"},{"company":"tess","family":"Tess","model":"tess-v2-5.2-qwen2-72b","place":120,"score":16.6,"version":"Tess"},{"company":"mistral","family":"Mistral","model":"mistral-nemo-12b-instruct","place":121,"score":16.6,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen1.5-110b-chat","place":122,"score":15.9,"version":"Qwen1.5"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-6.7b","place":123,"score":15.2,"version":"OpenCodeInterpreter DS"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-ds-6.7b","place":124,"score":15.2,"version":"Reflectioncoder"},{"company":"IBM","family":"Granite","model":"granite-code-34b-instruct","place":125,"score":14.8,"version":"Granite 3.1"},{"company":"wavecoder","family":"WaveCoder-Ultra-6.7B","model":"wavecoder-ultra-6.7b","place":126,"score":14.8,"version":"WaveCoder-Ultra-6.7B"},{"company":"microsoft","family":"Phi","model":"phi-3.5-mini-instruct","place":127,"score":14.8,"version":"Phi 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-instruct-2409","place":128,"score":14.8,"version":"Mistral"},{"company":"01ai","family":"Yi 1.5","model":"yi-coder-9b-chat","place":129,"score":14.6,"version":"Yi 1.5"},{"company":"artigenz","family":"Artigenz-Coder-DS-6.7B","model":"artigenz-coder-ds-6.7b","place":130,"score":14.2,"version":"Artigenz Coder DS"},{"company":"phind","family":"Phind-CodeLlama-34B-v2","model":"phind-codellama-34b-v2","place":131,"score":14.2,"version":"Phind-CodeLlama-34B-v2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":132,"score":14.2,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-2402","place":133,"score":13.9,"version":"Mistral"},{"company":"mistral","family":"Codestral","model":"codestral-mamba","place":134,"score":13.9,"version":"Codestral"},{"company":"inverse","family":"Inversecoder","model":"inversecoder-cl-13b","place":135,"score":13.9,"version":"Inversecoder"},{"company":"cohere","family":"Command","model":"command-r-plus","place":136,"score":13.8,"version":"Command"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b-instruct-v0.1","place":137,"score":13.6,"version":"StarCoder2"},{"company":"codellama","family":"CodeLlama","model":"codellama-70b-instruct","place":138,"score":13.5,"version":"CodeLlama"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":139,"score":13.5,"version":"Llama 3.1"},{"company":"inverse","family":"Inversecoder","model":"inversecoder-cl-7b","place":140,"score":13.5,"version":"Inversecoder"},{"company":"cohere","family":"C4ai","model":"c4ai-command-r-08.2024","place":141,"score":13.5,"version":"C4ai"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":142,"score":13.5,"version":"Qwen2.5"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":143,"score":13.2,"version":"Magicoder-S-DS-6.7B"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-128k-instruct","place":144,"score":13.2,"version":"Phi 3.5"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-cl-34b","place":145,"score":13.2,"version":"Reflectioncoder"},{"company":"cohere","family":"C4ai","model":"c4ai-command-r-plus-08.2024","place":146,"score":13.2,"version":"C4ai"},{"company":"inverse","family":"Inversecoder","model":"inversecoder-ds-6.7b","place":147,"score":12.9,"version":"Inversecoder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":148,"score":12.8,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen2.57b-a14b","place":149,"score":12.5,"version":"Qwen2"},{"company":"microsoft","family":"Phi","model":"phi-3-small-128k-instruct","place":150,"score":12.5,"version":"Phi 3.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-72b-chat","place":151,"score":12.2,"version":"Qwen1.5"},{"company":"qwen","family":"Qwen","model":"qwen2-7b-instruct","place":152,"score":11.8,"version":"Qwen2"},{"company":"openchat","family":"OpenChat-3.5-7B-0106","model":"openchat-3.6-8b-20240522","place":153,"score":11.5,"version":"OpenChat-3.5-7B-0106"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":154,"score":11.2,"version":"Qwen1.5"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-9b-chat","place":155,"score":11.1,"version":"Yi 1.5"},{"company":"IBM","family":"Granite","model":"granite-code-20b-instruct","place":156,"score":10.8,"version":"Granite 3.1"},{"company":"autocoder","family":"Autocoder","model":"autocoder-qw-7b","place":157,"score":10.8,"version":"Autocoder"},{"company":"microsoft","family":"Phi","model":"phi-3-mini-128k-instruct","place":158,"score":10.8,"version":"Phi 3.5"},{"company":"autocoder","family":"Autocoder","model":"autocoder-s-6.7b","place":159,"score":10.5,"version":"Autocoder"},{"company":"codegemma","family":"codegemma-2b","model":"codegemma-7b-instruct","place":160,"score":10.4,"version":"Codegemma"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":161,"score":10.1,"version":"Gemma 2"},{"company":"meta","family":"CodeLlama","model":"codellama-34b-instruct","place":162,"score":9.8,"version":"CodeLlama"},{"company":"reflection","family":"Reflectioncoder","model":"reflectioncoder-cl-7b","place":163,"score":9.4,"version":"Reflectioncoder"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-34b-chat","place":164,"score":9.2,"version":"Yi 1.5"},{"company":"map","family":"OpenCoder","model":"opencoder-1.5b-instruct","place":165,"score":9.2,"version":"OpenCoder-8B-Instruct"},{"company":"mistral","family":"Mistral","model":"mistral-large-2402","place":166,"score":8.4,"version":"Mistral"},{"company":"IBM","family":"Granite","model":"granite-3.0-8b-instruct","place":167,"score":8.4,"version":"Granite 3.1"},{"company":"meta","family":"CodeLlama","model":"codellama-13b-instruct","place":168,"score":8.2,"version":"CodeLlama"},{"company":"meta","family":"Llama","model":"llama-3.8b-instruct","place":169,"score":7.4,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":170,"score":6.1,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-1.5b-instruct","place":171,"score":5.8,"version":"Qwen2.5"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-6b-chat","place":172,"score":5.4,"version":"Yi 1.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.3","place":173,"score":5.4,"version":"Mistral 0.3"},{"company":"internlm","family":"Intern","model":"internlm2.5.7b-chat","place":174,"score":5.4,"version":"Internlm2.5.7b"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":175,"score":5.4,"version":"Llama 3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":176,"score":4.8,"version":"DeepSeek R1"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-1.3b","place":177,"score":4.4,"version":"OpenCodeInterpreter DS"},{"company":"meta","family":"CodeLlama","model":"codellama-7b-instruct","place":178,"score":3.8,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-instruct","place":179,"score":2.7,"version":"DeepSeek Coder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":180,"score":2.7,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b-instruct","place":181,"score":1.4,"version":"Qwen2.5"}],"stars":"","top_company":"anthropic","top_model":"Claude 3.7 Sonnet (2025-02-19)","top_model_family":"Claude","top_model_id":"claude-3.7-sonnet-20250219","top_model_version":"Claude 3.7","top_open_source":0,"top_percent":35.8,"topic":"Code generation (hard)","total_models":1613,"url":"https://bigcode-project.github.io/bigcodebench/"},{"bench_models":5,"description":"Biology knowledge and reasoning benchmark.","github":"","has_recent":0,"id":"biobench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BIOBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":51.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":48,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":44.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":42.3,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":41.5,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":51.9,"topic":"Biology reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Multi-omics sequence reasoning benchmark for biological data understanding.","github":"","has_recent":0,"id":"biology-instruction","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Biology-Instruction","paper":"","placements":[{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":1,"score":52.5,"version":"Intern-S1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":12,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":10.7,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":10.2,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":6.2,"version":"Qwen3 VL"}],"stars":"","top_company":"internlm","top_model":"Intern-S1-Pro","top_model_family":"Intern","top_model_id":"intern-s1-pro","top_model_version":"Intern-S1","top_open_source":1,"top_percent":52.5,"topic":"Biology multi-omics","total_models":1613,"url":""},{"bench_models":3,"description":"Comprehensive biomedical language processing benchmark evaluating LLMs across tasks like NER, relation extraction, and QA.","github":"https://github.com/baceolus/BioLP-bench","has_recent":0,"id":"biolp-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BioLP-Bench","paper":"","placements":[{"company":"xai","family":"Grok","model":"grok-4","place":1,"score":47,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":2,"score":37,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":3,"score":19.9,"version":"Grok Code"}],"stars":"","top_company":"xai","top_model":"Grok 4","top_model_family":"Grok","top_model_id":"grok-4","top_model_version":"Grok 4","top_open_source":0,"top_percent":47,"topic":"Biomedical NLP","total_models":1613,"url":"https://github.com/baceolus/BioLP-bench"},{"bench_models":22,"description":"Natural language to SQL generation benchmark.","github":"","has_recent":0,"id":"bird-sql","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Bird-SQL","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":1,"score":59.3,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":2,"score":58.7,"version":"Gemini 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":3,"score":54.4,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":4,"score":54.4,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":5,"score":54.4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":6,"score":47.9,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":7,"score":47.75,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":8,"score":47.49,"version":"Ling 2.0"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":9,"score":46.7,"version":"Gemma 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":10,"score":45.76,"version":"LLaDA 2.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":11,"score":45.6,"version":"Gemini 1.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":12,"score":44.04,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":13,"score":42.18,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":14,"score":39.67,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":15,"score":39.34,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":16,"score":38.4,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":17,"score":37.32,"version":"LLaDA 2.1"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":18,"score":36.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":19,"score":36.11,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":20,"score":33.8,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":21,"score":12.2,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":22,"score":6.4,"version":"Gemma 3"}],"stars":"","top_company":"google","top_model":"Gemini 2.0 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.0-pro","top_model_version":"Gemini 2.0","top_open_source":0,"top_percent":59.3,"topic":"Text-to-SQL","total_models":1613,"url":""},{"bench_models":64,"description":"Evaluates visual-language grounding and reference resolution to reduce hallucinations.","github":"","has_recent":0,"id":"blink","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BLINK","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":87.4,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":77.1,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":74.3,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":73.7,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":5,"score":72.38,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":72.1,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":71.86,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":8,"score":71,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":70.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":10,"score":70.2,"version":"Gemini 2.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":11,"score":70.2,"version":"Ovis 2.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":69.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":13,"score":69.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":14,"score":68.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":15,"score":67.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":16,"score":67.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":17,"score":67.1,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":18,"score":66.79,"version":"Step3-VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":19,"score":66.5,"version":"GPT-5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":20,"score":66.33,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":21,"score":65.8,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":22,"score":65.5,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":23,"score":65.5,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":24,"score":65.3,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":25,"score":65.1,"version":"GLM 4.1V"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":26,"score":65.1,"version":"GLM 4.1V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":27,"score":64.9,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":28,"score":64.8,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":29,"score":64.4,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":30,"score":64.1,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":31,"score":63.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":32,"score":63.4,"version":"GPT-5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":33,"score":62.8,"version":"ERNIE 4.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":34,"score":62.78,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step-3","place":35,"score":62.7,"version":"Step 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":36,"score":62.57,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":37,"score":62.5,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":38,"score":62.4,"version":"MiMo VL"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":39,"score":62.3,"version":"Claude 3.7"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":40,"score":62,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":41,"score":61.7,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":42,"score":60,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":43,"score":59.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":44,"score":59.9,"version":"Claude 4"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":45,"score":59.5,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":46,"score":58.1,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":47,"score":57.5,"version":"Molmo2"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":48,"score":56.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":49,"score":56.4,"version":"Qwen2.5 VL"},{"company":"unknown","family":"PLM","model":"plm-8b","place":50,"score":56,"version":"PLM"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":51,"score":55.5,"version":"InternVL"},{"company":"unknown","family":"PLM","model":"plm-3b","place":52,"score":55.4,"version":"PLM"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":53,"score":55.4,"version":"InternVL 3.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":54,"score":54.9,"version":"Keye-VL 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":55,"score":53.8,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":56,"score":53.5,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":57,"score":53.3,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":58,"score":53.1,"version":"InternVL 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":59,"score":52.9,"version":"Gemma 3"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":60,"score":51.3,"version":"Molmo2"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":61,"score":51.03,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":62,"score":50.97,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":63,"score":50.5,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":64,"score":48.97,"version":"Qwen2.5 VL"},{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":65,"score":48.82,"version":"LFM2.5-VL"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":66,"score":45.8,"version":"Eagle 2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":67,"score":44.5,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":68,"score":44.19,"version":"InternVL 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":69,"score":43.29,"version":"FastVLM"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":70,"score":42.3,"version":"SmolVLM2"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":71,"score":42.2,"version":"GPT-5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":72,"score":42,"version":"MiniCPM V 4.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":87.4,"topic":"Multimodal grounding","total_models":1613,"url":""},{"bench_models":32,"description":"Hard, Versatile, and Relevant composite score across eight capability buckets.","github":"","has_recent":0,"id":"bob-hvr","leaderboard":"","lower_is_better":false,"metric":"index (0-100)","name":"BoB-HVR","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3-70b","place":1,"score":8.98,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":2,"score":3.02,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":3,"score":2.79,"version":"DeepSeek V3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":4,"score":2.77,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":5,"score":2.21,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":6,"score":2.05,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":7,"score":2.03,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-256b","place":8,"score":1.95,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4","place":9,"score":1.93,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonet","place":10,"score":1.88,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":11,"score":1.85,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-3","place":12,"score":1.77,"version":"GPT-3"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":13,"score":1.63,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4","place":14,"score":1.59,"version":"Grok 4"},{"company":"moonshot","family":"Kimi","model":"kimi-k20895","place":15,"score":1.52,"version":"Kimi K2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":16,"score":1.52,"version":"Llama 3.3"},{"company":"openai","family":"o3","model":"gpt-o3","place":17,"score":1.48,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4-flash","place":18,"score":1.41,"version":"GLM 4"},{"company":"xai","family":"Grok","model":"grok-core-poset","place":19,"score":1.41,"version":"Grok Core"},{"company":"openai","family":"GPT","model":"gpt-5","place":20,"score":1.36,"version":"GPT-5"},{"company":"meta","family":"Llama","model":"llama-4-memonet","place":21,"score":1.34,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":22,"score":1.26,"version":"Llama 3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking","place":23,"score":1.23,"version":"Kimi VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":24,"score":1.19,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":25,"score":1.15,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":26,"score":1.08,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":27,"score":1.08,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-4","place":28,"score":0.99,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-super-105-turbo","place":29,"score":0.73,"version":"GPT Super 105"},{"company":"openai","family":"GPT","model":"gpt-2","place":30,"score":0.54,"version":"GPT-2"},{"company":"mistral","family":"Mistral","model":"mistra-medium","place":31,"score":0.54,"version":"Mistral"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":32,"score":0.41,"version":"EXAONE 4.0"}],"stars":"","top_company":"meta","top_model":"Llama 3 70B","top_model_family":"Llama","top_model_id":"llama-3-70b","top_model_version":"Llama 3","top_open_source":1,"top_percent":8.98,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":2,"description":"Bias in Open-ended Language Dataset probing demographic biases in text generation.","github":"https://github.com/amazon-science/bold","has_recent":0,"id":"bold","leaderboard":"","lower_is_better":true,"metric":"score","name":"BOLD","paper":"https://arxiv.org/abs/2101.11718","placements":[{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":1,"score":0.084,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":2,"score":0.094,"version":"Llama 2"}],"stars":"","top_company":"mistral","top_model":"Mixtral 8x 7B","top_model_family":"Mixtral","top_model_id":"mixtral-8x7b","top_model_version":"Mixtral","top_open_source":1,"top_percent":0.084,"topic":"Bias evaluation","total_models":1613,"url":"https://huggingface.co/datasets/AmazonScience/bold"},{"bench_models":40,"description":"Yes/no QA from naturally occurring questions.","github":"https://github.com/google-research-datasets/boolean-questions","has_recent":0,"id":"boolq","leaderboard":"https://llm-evals.formula-labs.com/","lower_is_better":false,"metric":"percent","name":"BoolQ","paper":"https://arxiv.org/pdf/1905.10044","placements":[{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":1,"score":89.4,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":2,"score":88.9,"version":"Marin 32B"},{"company":"google","family":"PaLM","model":"palm-540b","place":3,"score":88,"version":"PaLM"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":4,"score":87.7,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":5,"score":87.1,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":6,"score":85.3,"version":"Llama 1"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":7,"score":84.8,"version":"Gemma 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":8,"score":84.8,"version":"Nemotron Nano v2"},{"company":"google","family":"PaLM","model":"palm-62b","place":9,"score":84.8,"version":"PaLM"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":10,"score":84.6,"version":"Phi 3.5 MoE"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":11,"score":84.2,"version":"Gemma 2"},{"company":"google","family":"PaLM","model":"palm-cont-62b","place":12,"score":83.9,"version":"PaLM"},{"company":"google","family":"Chinchilla","model":"chinchilla-70b","place":13,"score":83.7,"version":"Chinchilla"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":14,"score":83.2,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":15,"score":83.1,"version":"Llama 1"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":16,"score":83,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":17,"score":82.4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3n-e4b","place":18,"score":81.6,"version":"Gemma 3n"},{"company":"google","family":"Gemma","model":"gemma-3n-e4b-instructed-litert-preview","place":19,"score":81.6,"version":"Gemma 3n"},{"company":"microsoft","family":"Phi","model":"phi-4-mini","place":20,"score":81.2,"version":"Phi 4"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":21,"score":80,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":22,"score":79.4,"version":"Llama 3.1"},{"company":"google","family":"Gopher","model":"gopher-280b","place":23,"score":79.3,"version":"Gopher"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":24,"score":79,"version":"Llama 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":25,"score":78.8,"version":"Gemma 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":26,"score":78.5,"version":"Motif 2"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":27,"score":78.1,"version":"Llama 1"},{"company":"microsoft","family":"Phi","model":"phi-3.5-mini-instruct","place":28,"score":78,"version":"Phi 3.5"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":29,"score":76.5,"version":"Llama 1"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b","place":30,"score":76.4,"version":"Gemma 3n"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-instructed-litert-preview","place":31,"score":76.4,"version":"Gemma 3n"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":32,"score":76.24,"version":"MobileLLM P1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":33,"score":75.7,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":34,"score":75,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":35,"score":69.4,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":36,"score":63.2,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":37,"score":62.51,"version":"Llama 3.2"},{"company":"loka","family":"TRLM","model":"trlm-135m","place":38,"score":62.17,"version":"TRLM"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":39,"score":61.4,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-3","place":40,"score":60.5,"version":"GPT-3"}],"stars":"171","top_company":"marin","top_model":"Marin-32B-Mantis","top_model_family":"Marin","top_model_id":"marin-32b-mantis","top_model_version":"Marin 32B","top_open_source":1,"top_percent":89.4,"topic":"Reading comprehension","total_models":1613,"url":"https://github.com/google-research-datasets/boolean-questions"},{"bench_models":8,"description":"Borda count aggregate ranking across multilingual benchmarks; lower is better.","github":"","has_recent":0,"id":"borda-count-multilingual","leaderboard":"","lower_is_better":true,"metric":"borda_count","name":"Borda Count (Multilingual)","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":1,"score":2.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":2,"score":2.9,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":3,"score":3.3,"version":"Llama 3.3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":4,"score":3.7,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":5,"score":4.7,"version":"Mistral 3.2"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":6,"score":4.9,"version":"EuroLLM"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":7,"score":6.6,"version":"Apertus"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":8,"score":7.1,"version":"OLMo 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-32B","top_model_family":"Qwen","top_model_id":"qwen3-32b","top_model_version":"Qwen3","top_open_source":1,"top_percent":2.9,"topic":"Aggregate ranking","total_models":1613,"url":""},{"bench_models":4,"description":"BridgeBench evaluation benchmark.","github":"","has_recent":0,"id":"bridgebench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BridgeBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":60.1,"version":"Claude 4.6"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":2,"score":59.7,"version":"MiniMax M2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":3,"score":58.3,"version":"GPT-5.2 Codex"},{"company":"zai","family":"GLM","model":"glm-5","place":4,"score":41.5,"version":"GLM 5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":60.1,"topic":"Reasoning","total_models":1613,"url":""},{"bench_models":62,"description":"Web browsing comprehension and competence benchmark.","github":"","has_recent":0,"id":"browsecomp","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BrowseComp","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":85.9,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":84,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":3,"score":84,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":77.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":77.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":6,"score":74.7,"version":"Claude 4.6"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-235b","place":7,"score":69.8,"version":"MiroThinker v1.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":8,"score":69,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"chatgpt-agent","place":9,"score":68.9,"version":"ChatGPT"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":67.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":67.8,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":12,"score":67.6,"version":"Seed 1.8"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":13,"score":67.6,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":14,"score":67.5,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":15,"score":65.8,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":16,"score":65.8,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":17,"score":65.8,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":18,"score":63.8,"version":"Qwen3.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":19,"score":62,"version":"MiniMax M2.1"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":20,"score":61,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":21,"score":61,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":22,"score":60.6,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":23,"score":60.6,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":24,"score":60.2,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":25,"score":59.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-deepresearch","place":26,"score":59.2,"version":"Gemini DeepResearch"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":27,"score":58.3,"version":"MiMo V2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":28,"score":56.6,"version":"LongCat-Flash"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-30b","place":29,"score":56.1,"version":"MiroThinker v1.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":30,"score":54.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":31,"score":54.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":32,"score":54.9,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":33,"score":53.9,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":34,"score":52,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":35,"score":52,"version":"GLM 4.7"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":36,"score":51.6,"version":"Step 3.5 Flash"},{"company":"openai","family":"DeepResearch","model":"openai-deepresearch","place":37,"score":51.5,"version":"DeepResearch"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":38,"score":51.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":39,"score":51.4,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":40,"score":50.8,"version":"GPT-5.1"},{"company":"openai","family":"o3","model":"o3","place":41,"score":49.7,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":42,"score":48.1,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":43,"score":47.4,"version":"MiniMax M2.1"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.0-72b","place":44,"score":47.1,"version":"MiroThinker v1.0"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":45,"score":45.4,"version":"MiMo V2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":46,"score":45.1,"version":"GLM 4.6"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":47,"score":44,"version":"MiniMax M2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":48,"score":44,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":49,"score":43.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":50,"score":43.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":51,"score":43.9,"version":"Claude 4.5"},{"company":"tongyi","family":"Tongyi","model":"tongyi-deepresearch","place":52,"score":43.4,"version":"Tongyi"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":53,"score":42.8,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":54,"score":41.5,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":55,"score":41.1,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":56,"score":40.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":57,"score":40.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":58,"score":38.5,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":59,"score":37.8,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":60,"score":37,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":61,"score":37,"version":"Claude 4.5"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":62,"score":31.4,"version":"K-EXAONE"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":63,"score":30,"version":"DeepSeek V3.1"},{"company":"openai","family":"o4","model":"o4-mini","place":64,"score":28.3,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":65,"score":28.3,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5","place":66,"score":26.4,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":67,"score":24.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":68,"score":24.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":69,"score":24.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":70,"score":19.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":71,"score":19.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":72,"score":19.6,"version":"Claude 4.5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v0.2-32b","place":73,"score":17.2,"version":"MiroThinker v0.2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":74,"score":14.7,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":75,"score":14.1,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":76,"score":14.1,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":77,"score":12.2,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":78,"score":9.9,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":79,"score":7.6,"version":"GLM 4.6V"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":80,"score":7.4,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":81,"score":7.1,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":82,"score":2.29,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":85.9,"topic":"Web browsing","total_models":1613,"url":"https://openai.com/index/browsecomp/"},{"bench_models":19,"description":"BrowseComp benchmark evaluated with content manager assistance.","github":"","has_recent":0,"id":"browsecomp-with-content-manager","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BrowseComp (With Content Manager)","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":84,"version":"Claude 4.6"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":2,"score":76.3,"version":"MiniMax M2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":74.9,"version":"Kimi K2.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":4,"score":73.1,"version":"LongCat-Flash"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":5,"score":70,"version":"GPT-5.2"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":6,"score":69,"version":"Step 3.5 Flash"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":67.8,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":8,"score":67.6,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":9,"score":67.6,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":10,"score":67.5,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":11,"score":67.5,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":12,"score":65.8,"version":"GPT-5.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":13,"score":62,"version":"MiniMax M2.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":14,"score":62,"version":"MiniMax M2.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":15,"score":60.2,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":59.2,"version":"Gemini 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":17,"score":58.3,"version":"MiMo V2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":18,"score":57.8,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":19,"score":57.5,"version":"GLM 4.6"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":20,"score":56.9,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":21,"score":26.1,"version":"Claude 4.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":84,"topic":"Web browsing","total_models":1613,"url":"https://openai.com/index/browsecomp/"},{"bench_models":46,"description":"Chinese variant of the BrowseComp web browsing benchmark.","github":"","has_recent":0,"id":"browsecomp-zh","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BrowseComp_zh","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":81.3,"version":"Seed 1.8"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":2,"score":77.7,"version":"LongCat-Flash"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":76.1,"version":"GPT-5.2"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-235b","place":4,"score":71.5,"version":"MiroThinker v1.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":5,"score":70.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":6,"score":69.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":7,"score":69.5,"version":"Qwen3.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":8,"score":69,"version":"LongCat-Flash"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":9,"score":66.9,"version":"Step 3.5 Flash"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-30b","place":10,"score":66.8,"version":"MiroThinker v1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":11,"score":66.8,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":12,"score":66.6,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":13,"score":66.6,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":14,"score":65,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":15,"score":65,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":16,"score":65,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5","place":17,"score":63,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":18,"score":63,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":19,"score":62.4,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":20,"score":62.3,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":21,"score":62.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":22,"score":62.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":23,"score":60.9,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3","place":24,"score":58.1,"version":"o3"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.0-72b","place":25,"score":55.6,"version":"MiroThinker v1.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":26,"score":51.6,"version":"Gemini 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":27,"score":51.2,"version":"MiMo V2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":28,"score":49.5,"version":"GLM 4.6"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":29,"score":49.5,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":30,"score":49.2,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":31,"score":49.2,"version":"DeepSeek V3.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":32,"score":48.5,"version":"MiniMax M2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":33,"score":47.9,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":34,"score":47.9,"version":"DeepSeek V3.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":35,"score":47.8,"version":"MiniMax M2.1"},{"company":"tongyi","family":"Tongyi","model":"tongyi-deepresearch","place":36,"score":46.7,"version":"Tongyi"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":37,"score":45,"version":"DeepSeek V3.1"},{"company":"openai","family":"DeepResearch","model":"openai-deepresearch","place":38,"score":42.9,"version":"DeepResearch"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":39,"score":42.9,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":40,"score":42.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":41,"score":42.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":42,"score":42.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":43,"score":40.8,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":44,"score":37.5,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":45,"score":34.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":46,"score":32.2,"version":"Gemini 2.5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v0.2-32b","place":47,"score":29.4,"version":"MiroThinker v0.2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":48,"score":29.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":49,"score":29.1,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":50,"score":28.8,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":51,"score":28.8,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":52,"score":22.2,"version":"Kimi K2"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":81.3,"topic":"Web browsing (Chinese)","total_models":1613,"url":""},{"bench_models":4,"description":"BruMo 2025 olympiad-style mathematics benchmark.","github":"","has_recent":0,"id":"brumo25","leaderboard":"","lower_is_better":false,"metric":"percent","name":"BRuMo25","paper":"","placements":[{"company":"amazon","family":"Nemotron","model":"questa-nemotron-1.5b","place":1,"score":69.48,"version":"QuestA Nemotron"},{"company":"deepseek","family":"DeepSeek-R1","model":"deepseek-r1-distill-32b","place":2,"score":68,"version":"32B"},{"company":"nvidia","family":"Nemotron","model":"nemotron-1.5b","place":3,"score":58.23,"version":"Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":4,"score":44.06,"version":"Qwen3"}],"stars":"","top_company":"amazon","top_model":"QuestA Nemotron 1.5B","top_model_family":"Nemotron","top_model_id":"questa-nemotron-1.5b","top_model_version":"QuestA Nemotron","top_open_source":1,"top_percent":69.48,"topic":"Math competition","total_models":1613,"url":""},{"bench_models":10,"description":"A humour analysis benchmark.","github":"https://github.com/EQ-bench","has_recent":0,"id":"buzzbench","leaderboard":"https://eqbench.com/buzzbench.html","lower_is_better":false,"metric":"percent","name":"BuzzBench","paper":"https://arxiv.org/abs/2312.06281","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":71.09,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":2,"score":69.11,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":3,"score":68.2,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":4,"score":67.1,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":5,"score":62.73,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":6,"score":61.94,"version":"Claude 3.5"},{"company":"openai","family":"o1","model":"o1","place":7,"score":59.28,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":8,"score":54.9,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":9,"score":54.39,"version":"Gemini 2.0"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":10,"score":53.34,"version":"Claude 3.5"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":71.09,"topic":"Humor analysis","total_models":1613,"url":""},{"bench_models":48,"description":"Chinese college-level exam benchmark.","github":"","has_recent":0,"id":"c-eval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"C-Eval","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":1,"score":94,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":2,"score":93.7,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":93.4,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":93,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":5,"score":92.9,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":6,"score":92.5,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":7,"score":92.2,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":92.2,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":9,"score":92.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":10,"score":91.9,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":11,"score":91,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":12,"score":90.91,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":13,"score":90.7,"version":"Qwen2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":14,"score":90.6,"version":"ERNIE 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":15,"score":90.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":16,"score":90.5,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":17,"score":90.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":18,"score":90.2,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":19,"score":90.1,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":20,"score":90,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":21,"score":88.17,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":22,"score":88.12,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":23,"score":88.03,"version":"Ling 2.0"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":24,"score":87.9,"version":"MiMo V2"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":25,"score":87.54,"version":"Ling 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":26,"score":87.3,"version":"DeepSeek V3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":27,"score":86.93,"version":"LLaDA 2.1"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":28,"score":86.9,"version":"GLM 4.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":29,"score":86.75,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":30,"score":86.71,"version":"LLaDA 2.1"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":31,"score":86.55,"version":"LongCat-Flash-Lite"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":32,"score":86.1,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":33,"score":85.21,"version":"LLaDA 2.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":34,"score":83.3,"version":"Kimi Linear"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":35,"score":82.2,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":36,"score":82.17,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":37,"score":81.8,"version":"LLaDA 2.0"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":38,"score":80.9,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":39,"score":80.6,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":40,"score":78.59,"version":"LLaDA 2.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":41,"score":78.48,"version":"Kimi Linear"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":42,"score":78.4,"version":"LLaDA 2.1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":43,"score":77.9,"version":"GPT-4.1"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":44,"score":77.6,"version":"Moonlight"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":45,"score":76.2,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":46,"score":75.16,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":47,"score":72.5,"version":"Llama 3.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":48,"score":66.54,"version":"LLaDA 2.0"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":49,"score":54.27,"version":"HY"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":50,"score":48.67,"version":"HY"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":51,"score":47.6,"version":"HY"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":52,"score":35.98,"version":"HY"}],"stars":"1768","top_company":"moonshotai","top_model":"Kimi-K2.5","top_model_family":"Kimi","top_model_id":"kimi-k2.5","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":94,"topic":"Chinese exams","total_models":1613,"url":"https://github.com/SJTU-LIT/ceval"},{"bench_models":3,"description":"Comprehensive Chinese reasoning capability benchmark.","github":"","has_recent":0,"id":"c3-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"C3-Bench","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.5-base","place":1,"score":83.1,"version":"GLM 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":2,"score":78.6,"version":"DeepSeek V3"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-7b-instruct","place":3,"score":68.5,"version":"Hunyuan"}],"stars":"35","top_company":"zai","top_model":"GLM-4.5 Base","top_model_family":"GLM","top_model_id":"glm-4.5-base","top_model_version":"GLM 4.5","top_open_source":1,"top_percent":83.1,"topic":"Reasoning (Chinese)","total_models":1613,"url":"https://github.com/Tencent-Hunyuan/C3-Benchmark"},{"bench_models":30,"description":"U.S. case law benchmark evaluating legal reasoning and judgment over court opinions.","github":"","has_recent":0,"id":"caselaw-v2","leaderboard":"https://www.vals.ai/benchmarks/case_law_v2-09-20-2025","lower_is_better":false,"metric":"percent","name":"CaseLaw v2","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-4.1","place":1,"score":78.1,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":2,"score":77.5,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4","place":3,"score":76.2,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-3","place":4,"score":75.2,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":5,"score":74.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":6,"score":74.6,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":7,"score":74,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":8,"score":74,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":9,"score":73.6,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":72.7,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":11,"score":72.7,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-4.1-opus-thinking","place":12,"score":72.3,"version":"Claude 4.1"},{"company":"xai","family":"Grok","model":"grok-4-fast-thinking","place":13,"score":71.4,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":14,"score":71.1,"version":"Claude 4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":15,"score":70.1,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":16,"score":70.1,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":17,"score":69.8,"version":"GPT-4o"},{"company":"openai","family":"o3","model":"o3","place":18,"score":69.5,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":19,"score":68.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":20,"score":66.6,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":21,"score":66.2,"version":"Claude 3.7"},{"company":"zai","family":"GLM","model":"glm-4.5","place":22,"score":66.2,"version":"GLM 4.5"},{"company":"xai","family":"Grok","model":"grok-3-mini-fast-low-reasoning","place":23,"score":65.9,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":24,"score":65.3,"version":"Qwen3"},{"company":"openai","family":"o4","model":"o4-mini","place":25,"score":64,"version":"o4"},{"company":"xai","family":"Grok","model":"grok-3-mini-fast-high-reasoning","place":26,"score":64,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":27,"score":63.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":28,"score":62.1,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":29,"score":53.4,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":30,"score":51.4,"version":"GPT-4.1"}],"stars":"","top_company":"openai","top_model":"GPT-4.1","top_model_family":"GPT","top_model_id":"gpt-4.1","top_model_version":"GPT-4.1","top_open_source":0,"top_percent":78.1,"topic":"Legal reasoning","total_models":1613,"url":"https://www.vals.ai/benchmarks/case_law_v2-09-20-2025"},{"bench_models":26,"description":"Cross-lingual OCR benchmark evaluating character recognition across mixed-language documents.","github":"","has_recent":0,"id":"cc-ocr","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CC-OCR","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":82,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":2,"score":81.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":3,"score":81.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":4,"score":81.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":5,"score":81,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":6,"score":80.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":7,"score":80.7,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":8,"score":80.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":9,"score":79.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":10,"score":79.8,"version":"Qwen2.5 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":11,"score":79.7,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":79,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":77.2,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":14,"score":76.9,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":15,"score":76.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":16,"score":72.8,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":17,"score":72.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":18,"score":70.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":19,"score":70.3,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":20,"score":69.1,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5","place":21,"score":68.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":22,"score":68.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":23,"score":67.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":24,"score":67.3,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":25,"score":61.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":26,"score":52.9,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":82,"topic":"OCR (cross-lingual)","total_models":1613,"url":""},{"bench_models":5,"description":"Contest-style coding evaluation with ELO-like scoring.","github":"","has_recent":0,"id":"cfeval","leaderboard":"","lower_is_better":false,"metric":"ELO","name":"CFEval","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":1,"score":2134,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":2,"score":2071,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":3,"score":2044,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":4,"score":1995,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":5,"score":1986,"version":"Qwen3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-235B-A22B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-235b-a22b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":2134,"topic":"Coding ELO / contest eval","total_models":1613,"url":""},{"bench_models":4,"description":"Cartoon/CG long video question answering benchmark.","github":"","has_recent":0,"id":"cgbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CGBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":64.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":64.5,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":62.4,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":57.4,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":64.6,"topic":"Long video QA","total_models":1613,"url":""},{"bench_models":14,"description":"Charades-STA temporal grounding (mIoU).","github":"","has_recent":0,"id":"charades-sta","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Charades-STA","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":1,"score":64,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":2,"score":63.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":3,"score":61.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":4,"score":60.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":5,"score":56,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":6,"score":55.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":7,"score":54.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":8,"score":50.9,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":9,"score":50.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":10,"score":50.6,"version":"Claude 4"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":11,"score":50,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":12,"score":43.6,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":13,"score":38.5,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":14,"score":25.4,"version":"InternVL"}],"stars":"","top_company":"bytedance","top_model":"Seed1.5-VL-Thinking","top_model_family":"Seed","top_model_id":"seed1.5-vl-thinking","top_model_version":"Seed 1.5","top_open_source":0,"top_percent":64,"topic":"Video grounding","total_models":1613,"url":"https://huggingface.co/datasets/VLM2Vec/Charades-STA"},{"bench_models":8,"description":"Large-scale curated collection of charts for evaluating parsing, grounding, and reasoning.","github":"https://github.com/Liyan06/ChartMuseum","has_recent":0,"id":"chartmuseum","leaderboard":"https://chartmuseum-leaderboard.github.io/","lower_is_better":false,"metric":"percent","name":"ChartMuseum","paper":"https://arxiv.org/abs/2505.13444","placements":[{"company":"openai","family":"GPT","model":"gpt-5-mini","place":1,"score":63.3,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":2,"score":58.4,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":3,"score":55.3,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":4,"score":49.8,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":5,"score":48.8,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":6,"score":40,"version":"Step 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":7,"score":33.6,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":8,"score":23.9,"version":"Gemma 3"}],"stars":"","top_company":"openai","top_model":"GPT-5 mini","top_model_family":"GPT","top_model_id":"gpt-5-mini","top_model_version":"GPT-5","top_open_source":0,"top_percent":63.3,"topic":"Chart understanding","total_models":1613,"url":"https://chartmuseum-leaderboard.github.io/"},{"bench_models":100,"description":"Visual question answering over charts and plots.","github":"","has_recent":0,"id":"chartqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ChartQA","paper":"","placements":[{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":1,"score":94.1,"version":"Keye-VL 1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":93.7,"version":"Gemini 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":3,"score":92.9,"version":"MiMo VL"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":4,"score":92.2,"version":"Claude 3.7"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":5,"score":91.7,"version":"MiMo VL"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":6,"score":90.8,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":7,"score":90.2,"version":"Qwen2.5 VL"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":8,"score":90,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":9,"score":90,"version":"Llama 4"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":10,"score":89.6,"version":"InternVL"},{"company":"openai","family":"GPT","model":"gpt-5","place":11,"score":89.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":12,"score":89.6,"version":"Qwen3 VL"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":13,"score":89,"version":"Manzano"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":14,"score":88.8,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":15,"score":88.8,"version":"Llama 4"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":16,"score":88.4,"version":"InternVL 2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":17,"score":88.3,"version":"Qwen2 VL"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":18,"score":88.2,"version":"Manzano"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":19,"score":88.2,"version":"GPT-5"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":20,"score":88.1,"version":"Pixtral"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":21,"score":88.1,"version":"Claude 4.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":22,"score":87.68,"version":"dots.vlm1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":23,"score":87.6,"version":"Claude 3.5"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":24,"score":87.5,"version":"Eagle 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":25,"score":87.4,"version":"Mistral 3.2"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":26,"score":87.4,"version":"MiniCPM V 4.5"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":27,"score":87.3,"version":"Molmo"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":28,"score":87.2,"version":"Gemini 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":29,"score":87.2,"version":"InternVL 2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":30,"score":87.1,"version":"ERNIE 4.5 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":31,"score":86.88,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":32,"score":86.7,"version":"GPT-4o"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":33,"score":86.7,"version":"InternVL 3.5"},{"company":"moondream","family":"Moondream","model":"moondream-9b-a2b","place":34,"score":86.6,"version":"Moondream 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":35,"score":86.24,"version":"Mistral 3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":36,"score":86.16,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":37,"score":86.1,"version":"Qwen2.5 VL"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":38,"score":86.1,"version":"Molmo2"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":39,"score":86,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":40,"score":86,"version":"Molmo2"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":41,"score":85.7,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o","place":42,"score":85.7,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":43,"score":85.5,"version":"Llama 3.2"},{"company":"unknown","family":"PLM","model":"plm-8b","place":44,"score":85.5,"version":"PLM"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":45,"score":85.4,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-base","place":46,"score":85.3,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":47,"score":85.3,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":48,"score":85.1,"version":"GPT-4o"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":49,"score":85,"version":"FastVLM"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":50,"score":84.9,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":51,"score":84.6,"version":"Qwen3 VL"},{"company":"unknown","family":"PLM","model":"plm-3b","place":52,"score":84.3,"version":"PLM"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":53,"score":84.1,"version":"Molmo"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":54,"score":84,"version":"InternVL 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":55,"score":84,"version":"Qwen2.5 VL"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":56,"score":83.7,"version":"LLaVA OneVision"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":57,"score":83.6,"version":"MM1.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":58,"score":83.4,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-base","place":59,"score":83.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-base","place":60,"score":83.4,"version":"Llama 4"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":61,"score":83.3,"version":"InternVL 2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":62,"score":83.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":63,"score":83,"version":"Qwen2 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":64,"score":82.7,"version":"Gemini 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":65,"score":82.6,"version":"Mistral 3"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":66,"score":81.9,"version":"jina-VLM"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":67,"score":81.8,"version":"Phi 3.5"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":68,"score":81.8,"version":"Pixtral"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":69,"score":81.7,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":70,"score":81.5,"version":"Qwen2.5 Omni"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":71,"score":81.4,"version":"Phi-3"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":72,"score":80.8,"version":"Claude 3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":73,"score":80.7,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":74,"score":80.4,"version":"Molmo"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":75,"score":80.2,"version":"InternVL 3"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":76,"score":80.1,"version":"FastVLM"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":77,"score":80,"version":"LLaVA OneVision"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":78,"score":79.5,"version":"Gemini 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":79,"score":79.2,"version":"InternVL 2.5"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":80,"score":78.5,"version":"Bagel"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":81,"score":78.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4v","place":82,"score":78.1,"version":"GPT-4V"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":83,"score":78,"version":"Gemma 3"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":84,"score":78,"version":"MolmoE"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-0827","place":85,"score":78,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":86,"score":77.2,"version":"Qwen3 VL"},{"company":"apple","family":"MM1","model":"mm1-30b","place":87,"score":76.9,"version":"MM1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":88,"score":76.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":89,"score":76.3,"version":"Gemini 2.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":90,"score":76,"version":"FastVLM"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":91,"score":75.6,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":92,"score":75.6,"version":"Cambrian"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":93,"score":74.2,"version":"MM1.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":94,"score":73.5,"version":"Qwen2 VL"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":95,"score":73.3,"version":"Cambrian 1"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":96,"score":70,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":97,"score":69.6,"version":"Claude 3"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":98,"score":66.9,"version":"LLaVA OneVision"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":99,"score":60,"version":"xGen-MM"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":100,"score":59.8,"version":"MiniCPM V 2.0"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":101,"score":51.9,"version":"Gemini Nano"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":102,"score":34,"version":"MolmoE"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":103,"score":33.7,"version":"PaliGemma"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":104,"score":25.8,"version":"Janus Pro"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":105,"score":23.4,"version":"Janus Pro"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":106,"score":18.2,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":107,"score":17.8,"version":"LLaVA 1.5"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":108,"score":0,"version":"BLIP-3"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":109,"score":0,"version":"BLIP-3O"},{"company":"baai","family":"Emu","model":"emu3-8b","place":110,"score":0,"version":"Emu3"}],"stars":"","top_company":"kuaishou","top_model":"Keye-VL-1.5-8B","top_model_family":"Keye-VL","top_model_id":"keye-vl-1.5-8b","top_model_version":"Keye-VL 1.5","top_open_source":1,"top_percent":94.1,"topic":"Chart understanding (VQA)","total_models":1613,"url":"https://github.com/vis-nlp/ChartQA"},{"bench_models":11,"description":"Professional-grade chart question answering with diverse chart types and complex reasoning.","github":"","has_recent":0,"id":"chartqa-pro","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ChartQA-Pro","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":69.5,"version":"Gemini 2.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":2,"score":66.8,"version":"Ovis 2.6"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":3,"score":65.5,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":4,"score":64.7,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":5,"score":64,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":6,"score":62.6,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":7,"score":59.5,"version":"GLM 4.1V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":8,"score":59.3,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step-3","place":9,"score":56.4,"version":"Step 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":10,"score":37.6,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":11,"score":23.7,"version":"Kimi VL"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":69.5,"topic":"Chart understanding (VQA)","total_models":1613,"url":""},{"bench_models":29,"description":"Scientific chart/table descriptive questions from arXiv PDFs.","github":"","has_recent":0,"id":"charxiv-dq","leaderboard":"https://charxiv.github.io/","lower_is_better":false,"metric":"percent","name":"CharXiv (DQ)","paper":"","placements":[{"company":"openai","family":"o3","model":"o3-high","place":1,"score":95,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":93.5,"version":"Gemini 2.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":3,"score":92.1,"version":"dots.vlm1"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":91.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":5,"score":90.5,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":90.3,"version":"Gemini 2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":7,"score":90.3,"version":"ERNIE 4.5 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":8,"score":89.6,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":9,"score":89.5,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":10,"score":88.2,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":11,"score":87.8,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":12,"score":87.8,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":13,"score":87.4,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":14,"score":87,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":15,"score":86.8,"version":"MiMo VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":16,"score":86.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":17,"score":85.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":18,"score":83,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":19,"score":78.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":20,"score":76.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":21,"score":73.9,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":22,"score":73.6,"version":"InternVL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":23,"score":73.5,"version":"Gemini 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":24,"score":65,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":25,"score":64.4,"version":"GPT-5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":26,"score":63.8,"version":"Gemma 3"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":27,"score":63.5,"version":"jina-VLM"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":28,"score":62.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":29,"score":55,"version":"Qwen2 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":30,"score":54.7,"version":"InternVL 3"}],"stars":"","top_company":"openai","top_model":"o3-high","top_model_family":"o3","top_model_id":"o3-high","top_model_version":"o3","top_open_source":0,"top_percent":95,"topic":"Chart description (PDF)","total_models":1613,"url":"https://charxiv.github.io/"},{"bench_models":64,"description":"Scientific chart/table reasoning questions from arXiv PDFs.","github":"","has_recent":0,"id":"charxiv-rq","leaderboard":"https://charxiv.github.io/","lower_is_better":false,"metric":"percent","name":"CharXiv (RQ)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":1,"score":82.1,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":2,"score":82.1,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":3,"score":82.1,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":82.1,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":81.4,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":81.1,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":81.1,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":8,"score":80.8,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":9,"score":80.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":10,"score":79.5,"version":"Qwen3.5"},{"company":"openai","family":"o3","model":"o3","place":11,"score":78.6,"version":"o3"},{"company":"openai","family":"o3","model":"o3-high","place":12,"score":78.6,"version":"o3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":13,"score":77.5,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":14,"score":77.5,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":15,"score":77.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":16,"score":77.2,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":17,"score":71.4,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":69.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":19,"score":69.5,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":20,"score":69.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":21,"score":68.6,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":22,"score":68.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":23,"score":68.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":24,"score":68.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":25,"score":68.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":26,"score":68.3,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":27,"score":67.9,"version":"Gemini 2.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":28,"score":67.7,"version":"Ovis 2.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":29,"score":67.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":30,"score":67.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1-thinking","place":31,"score":67,"version":"GPT-5.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":32,"score":66.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":33,"score":66.1,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":34,"score":64.4,"version":"dots.vlm1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":35,"score":63.7,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":36,"score":63.6,"version":"Claude 4.1"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":37,"score":63.4,"version":"Seed 1.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":38,"score":63.2,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":39,"score":63,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":40,"score":62.8,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":41,"score":60.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":42,"score":60.9,"version":"Claude 4"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":43,"score":60.2,"version":"Seed 1.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":44,"score":59.6,"version":"GLM 4.6V"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":45,"score":56.9,"version":"ERNIE 4.5 VL"},{"company":"tencent","family":"R","model":"r-4b-base","place":46,"score":56.8,"version":"R"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":47,"score":56.6,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":48,"score":56.5,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":49,"score":54.4,"version":"MiMo VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":50,"score":52,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":51,"score":49.7,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":52,"score":48.9,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":53,"score":48.9,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":54,"score":48.5,"version":"Qwen2.5 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":55,"score":47.7,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":56,"score":46.4,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":57,"score":44.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":58,"score":42.5,"version":"Qwen2.5 VL"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":59,"score":40,"version":"Keye-VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":60,"score":39.7,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":61,"score":39.6,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":62,"score":37.6,"version":"InternVL"},{"company":"opengvlab","family":"Intern","model":"internvl3-8b","place":63,"score":37.6,"version":"InternVL 3"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":64,"score":32.3,"version":"jina-VLM"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":65,"score":31.7,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":66,"score":31.6,"version":"InternVL 3.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":67,"score":29.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":68,"score":28.8,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":69,"score":28.3,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":70,"score":26.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":71,"score":23.3,"version":"Qwen2 VL"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 Thinking","top_model_family":"GPT","top_model_id":"gpt-5.2-thinking","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":82.1,"topic":"Chart reasoning (PDF)","total_models":1613,"url":"https://charxiv.github.io/"},{"bench_models":8,"description":"Chinese variant of the SimpleQA benchmark.","github":"","has_recent":0,"id":"chinese-simpleqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Chinese SimpleQA","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":1,"score":77.6,"version":"Kimi K2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":2,"score":77.1,"version":"ERNIE 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":3,"score":76.1,"version":"GLM 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":4,"score":72.1,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":5,"score":72,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":6,"score":64,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":63,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":8,"score":53.5,"version":"Llama 4"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2 Base","top_model_family":"Kimi","top_model_id":"kimi-k2-base","top_model_version":"Kimi K2","top_open_source":1,"top_percent":77.6,"topic":"QA (Chinese)","total_models":1613,"url":""},{"bench_models":9,"description":"Comprehensive long-context benchmark evaluating reasoning over extended contexts.","github":"","has_recent":0,"id":"cl-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CL-Bench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":1,"score":25.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":23.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":22.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":4,"score":20.8,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":5,"score":20,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":18.1,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":7,"score":16.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":15.6,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":14.8,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5 Mini High","top_model_family":"GPT","top_model_id":"gpt-5-mini-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":25.2,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":8,"description":"Korean long-form instruction-following benchmark.","github":"","has_recent":0,"id":"click","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CLIcK","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":1,"score":86.3,"version":"DeepSeek V3.2"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":2,"score":83.9,"version":"K-EXAONE"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":3,"score":81.3,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":4,"score":80.9,"version":"DeepSeek V3.1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":5,"score":78.8,"version":"EXAONE 4.0"},{"company":"zai","family":"GLM","model":"glm-4.6","place":6,"score":77.9,"version":"GLM 4.6"},{"company":"skt","family":"A.X","model":"ax-k1","place":7,"score":77.2,"version":"A.X"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":8,"score":74.6,"version":"GPT-OSS"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.2-Thinking","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.2-thinking","top_model_version":"DeepSeek V3.2","top_open_source":1,"top_percent":86.3,"topic":"Korean instruction following","total_models":1613,"url":""},{"bench_models":2,"description":"Safety benchmark that red-teams models with cloning-related misuse scenarios to measure compliance and refusal rates.","github":"","has_recent":0,"id":"cloningscenarios","leaderboard":"","lower_is_better":true,"metric":"percent","name":"CloningScenarios","paper":"","placements":[{"company":"xai","family":"Grok","model":"grok-4","place":1,"score":45,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":2,"score":46,"version":"Grok 4.1"}],"stars":"","top_company":"xai","top_model":"Grok 4","top_model_family":"Grok","top_model_id":"grok-4","top_model_version":"Grok 4","top_open_source":0,"top_percent":45,"topic":"Biosecurity refusal","total_models":1613,"url":""},{"bench_models":3,"description":"Chinese Winograd Schema-style coreference benchmark from CLUE.","github":"","has_recent":0,"id":"cluewsc","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CLUEWSC","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":1,"score":92.8,"version":"DeepSeek R1"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":2,"score":83.5,"version":"GLM 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":3,"score":82.7,"version":"DeepSeek V3"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek R1","top_model_family":"DeepSeek","top_model_id":"deepseek-r1","top_model_version":"DeepSeek R1","top_open_source":1,"top_percent":92.8,"topic":"Coreference reasoning (Chinese)","total_models":1613,"url":"https://www.cluebenchmarks.com/"},{"bench_models":16,"description":"Chinese mathematics benchmark.","github":"","has_recent":0,"id":"cmath","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CMath","paper":"","placements":[{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":1,"score":96.9,"version":"LLaDA 2.0"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":2,"score":96.7,"version":"ERNIE 4.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":3,"score":96.63,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":96.58,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":5,"score":96.54,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":6,"score":96.52,"version":"Ling 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":7,"score":96.4,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":8,"score":95.72,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":9,"score":95.7,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":10,"score":95.63,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":11,"score":95.42,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":12,"score":94.99,"version":"LLaDA 2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":13,"score":94.8,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":14,"score":92.8,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":15,"score":85.5,"version":"Kimi Linear"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":16,"score":79.6,"version":"Moonlight"}],"stars":"","top_company":"antgroup","top_model":"LLaDA2.0 Flash","top_model_family":"LLaDA","top_model_id":"llada2.0-flash","top_model_version":"LLaDA 2.0","top_open_source":1,"top_percent":96.9,"topic":"Math (Chinese)","total_models":1613,"url":""},{"bench_models":33,"description":"Chinese counterpart to MMLU.","github":"","has_recent":0,"id":"cmmlu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CMMLU","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":1,"score":91.9,"version":"Qwen2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":2,"score":90.9,"version":"Kimi K2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":3,"score":90.2,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":4,"score":89.9,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":5,"score":88.9,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":6,"score":88.8,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":7,"score":88.2,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":8,"score":87.5,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":9,"score":87.4,"version":"MiMo V2"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":10,"score":87.03,"version":"JoyAI-LLM"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":11,"score":86.59,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":12,"score":86.5,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":13,"score":86.36,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":14,"score":85.88,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":15,"score":85.13,"version":"LLaDA 2.0"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":16,"score":84.72,"version":"MiniCPM 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":17,"score":83.6,"version":"Qwen3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":18,"score":83.09,"version":"JoyAI-LLM"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":19,"score":82.48,"version":"LongCat-Flash-Lite"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":20,"score":81.68,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":21,"score":81.55,"version":"MiniCPM SALA"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":22,"score":78.1,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":23,"score":76.26,"version":"Kimi Linear"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":24,"score":75.85,"version":"GLM 4.7"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":25,"score":73.7,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":26,"score":72.06,"version":"Gemini 2.5"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":27,"score":71.74,"version":"Ministral 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":28,"score":67.53,"version":"LLaDA 2.0"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":29,"score":63.55,"version":"Falcon H1R"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":30,"score":61.59,"version":"Nemotron Nano V2"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":31,"score":55.07,"version":"HY"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":32,"score":50.8,"version":"HY"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":33,"score":49.32,"version":"HY"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":34,"score":37.08,"version":"HY"}],"stars":"781","top_company":"qwen","top_model":"Qwen2.5 Max","top_model_family":"Qwen","top_model_id":"qwen-2.5-max","top_model_version":"Qwen2.5","top_open_source":0,"top_percent":91.9,"topic":"Chinese multi-domain","total_models":1613,"url":"https://github.com/haonan-li/CMMLU"},{"bench_models":9,"description":"China National Mathematical Olympiad 2024 evaluation set.","github":"","has_recent":0,"id":"cnmo-2024","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CNMO 2024","paper":"","placements":[{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":1,"score":82.8,"version":"openPangu-R"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":2,"score":77.8,"version":"openPangu-R"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":3,"score":75,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":4,"score":74.7,"version":"DeepSeek V3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":5,"score":74.3,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":6,"score":60.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":7,"score":57.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":8,"score":56.6,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":9,"score":48.6,"version":"Qwen3"}],"stars":"","top_company":"freedomintelligence","top_model":"openPangu-R-72B-2512 Slow Thinking","top_model_family":"openPangu-R","top_model_id":"openpangu-r-72b-2512-slow-thinking","top_model_version":"openPangu-R","top_open_source":1,"top_percent":82.8,"topic":"Math (competition)","total_models":1613,"url":""},{"bench_models":64,"description":"Competitive programming performance on Codeforces problems (ELO).","github":"","has_recent":0,"id":"codeforces","leaderboard":"https://llmdb.com/benchmarks/codeforces","lower_is_better":false,"metric":"ELO","name":"Codeforces","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-deep-think","place":1,"score":3455,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":3148,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":3,"score":3020,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":2727,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":2726,"version":"Gemini 3"},{"company":"openai","family":"o4","model":"o4-mini","place":6,"score":2719,"version":"o4"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":2708,"version":"Gemini 3"},{"company":"openai","family":"o3","model":"o3","place":8,"score":2706,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":9,"score":2701,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":10,"score":2622,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":11,"score":2537,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":12,"score":2516,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":13,"score":2386,"version":"DeepSeek V3.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":14,"score":2233,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":15,"score":2160,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":16,"score":2157,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":17,"score":2146,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":18,"score":2121,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":19,"score":2100,"version":"Qwen3.5"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":20,"score":2092,"version":"Ring"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":21,"score":2091,"version":"DeepSeek V3.1"},{"company":"antgroup","family":"Ring","model":"ring-1t","place":22,"score":2088,"version":"Ring"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":23,"score":2073,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen-3","place":24,"score":2056,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":25,"score":2056,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":26,"score":2055,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":27,"score":2046,"version":"DeepSeek V3.1"},{"company":"openai","family":"o3","model":"o3-mini","place":28,"score":2036,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":29,"score":2029,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":30,"score":2028,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":31,"score":2001,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":32,"score":1985,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":33,"score":1977,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-thinking-high","place":34,"score":1918,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":35,"score":1901,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":36,"score":1899,"version":"Qwen3.5"},{"company":"openai","family":"o1","model":"o1","place":37,"score":1891,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":38,"score":1891,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":39,"score":1837,"version":"Gemini 2.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":40,"score":1701.4,"version":"openPangu-R"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":41,"score":1701,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":42,"score":1675,"version":"Gemini 2.5"},{"company":"openai","family":"o1","model":"o1-mini","place":43,"score":1650,"version":"o1"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":44,"score":1644,"version":"Seed 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":45,"score":1600,"version":"Ling 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":46,"score":1582,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":47,"score":1574,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":48,"score":1485,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":49,"score":1480,"version":"Claude 4.5"},{"company":"openai","family":"o1","model":"o1-preview","place":50,"score":1258,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-5","place":51,"score":1120,"version":"GPT-5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":52,"score":1044.5,"version":"openPangu-R"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-80b-a13b-instruct","place":53,"score":683,"version":"Hunyuan"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":54,"score":678,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":55,"score":605,"version":"Seed OSS"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":56,"score":94.69,"version":"Ring"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":57,"score":93.86,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":58,"score":93.01,"version":"Qwen3"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":59,"score":90.23,"version":"Ring 2.0"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":60,"score":89.9,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":61,"score":89.77,"version":"Qwen3 Next"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":62,"score":89.67,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":63,"score":86.84,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":64,"score":86.84,"version":"GPT-5"},{"company":"antgroup","family":"Ring","model":"ring-mini-sparse-2.0-exp","place":65,"score":85.77,"version":"Ring 2.0"},{"company":"antgroup","family":"Ring","model":"ring-mini-2.0","place":66,"score":84.8,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":67,"score":84.25,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":68,"score":83.78,"version":"Seed OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":69,"score":82.25,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":70,"score":81.59,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":71,"score":73.31,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":72,"score":59.5,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":73,"score":55.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":74,"score":51.6,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-nothinking-2504","place":75,"score":28.2,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-21b-a3b-pt","place":76,"score":21.7,"version":"ERNIE 4.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Deep Think","top_model_family":"Gemini","top_model_id":"gemini-3-deep-think","top_model_version":"Gemini 3","top_open_source":0,"top_percent":3455,"topic":"Competitive programming","total_models":1613,"url":"https://codeforces.com/"},{"bench_models":5,"description":"Code-focused instruction following benchmark.","github":"","has_recent":0,"id":"codeif-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CodeIF-Bench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":1,"score":50,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":2,"score":48,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":3,"score":46,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":4,"score":46,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":5,"score":42,"version":"LLaDA 2.1"}],"stars":"","top_company":"qwen","top_model":"Qwen3-8B Non-Thinking","top_model_family":"Qwen","top_model_id":"qwen3-8b-non-thinking","top_model_version":"Qwen3","top_open_source":1,"top_percent":50,"topic":"Code instruction following","total_models":1613,"url":""},{"bench_models":12,"description":"Comprehensive instruction-following evaluation suite.","github":"","has_recent":0,"id":"collie","leaderboard":"","lower_is_better":false,"metric":"percent","name":"COLLIE","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":99,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":98.4,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":3,"score":97.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":4,"score":96.9,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":5,"score":96.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":95,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":7,"score":94,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":8,"score":93.9,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":91.2,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":79.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":77.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":12,"score":61,"version":"GPT-4o"}],"stars":"55","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":99,"topic":"Instruction following","total_models":1613,"url":"https://collie-benchmark.github.io/"},{"bench_models":5,"description":"Hard subset of Collie instruction-following tasks.","github":"","has_recent":0,"id":"collie-hard","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Collie-Hard","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":99,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":95.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":77.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":72.6,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":69.5,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":99,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":19,"description":"Multiple-choice QA requiring commonsense knowledge.","github":"","has_recent":0,"id":"commonsenseqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CommonsenseQA","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":1,"score":88.5,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":2,"score":85.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":3,"score":85.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":4,"score":84.1,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":5,"score":83.8,"version":"Llama 3"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":6,"score":82.8,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":7,"score":82.3,"version":"Marin 32B"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":8,"score":78.6,"version":"OLMo 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":9,"score":76.1,"version":"Nemotron Nano v2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":10,"score":75,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":11,"score":73.4,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":12,"score":72.6,"version":"Llama 3"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":13,"score":71.3,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":14,"score":65.3,"version":"Gemma 2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":15,"score":43.6,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":16,"score":41.2,"version":"Llama 1"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":17,"score":38,"version":"SmolLM1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":18,"score":34.1,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":19,"score":33.9,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":20,"score":33.9,"version":"SmolLM2"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5 32B Base","top_model_family":"Qwen","top_model_id":"qwen2.5-32b-base","top_model_version":"Qwen2.5","top_open_source":1,"top_percent":88.5,"topic":"Commonsense QA","total_models":1613,"url":"https://www.tau-nlp.org/csqa"},{"bench_models":5,"description":"Complex workflow benchmark for economically valuable tasks.","github":"","has_recent":0,"id":"complex-workflow","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Complex Workflow","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":58.2,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":55.4,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":54.6,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":54.4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":53,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":58.2,"topic":"Complex workflows","total_models":1613,"url":""},{"bench_models":6,"description":"Choice of Plausible Alternatives.","github":"","has_recent":0,"id":"copa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"COPA","paper":"","placements":[{"company":"marin","family":"Marin","model":"marin-32b-bison","place":1,"score":94,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":2,"score":93,"version":"Marin 32B"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":3,"score":93,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":4,"score":93,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":5,"score":87,"version":"Qwen2.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":6,"score":85,"version":"Nemotron Nano v2"}],"stars":"","top_company":"marin","top_model":"Marin-32B-Bison","top_model_family":"Marin","top_model_id":"marin-32b-bison","top_model_version":"Marin 32B","top_open_source":1,"top_percent":94,"topic":"Causal reasoning","total_models":1613,"url":"https://people.ict.usc.edu/~gordon/copa.html"},{"bench_models":2,"description":"Comprehensive Ontological Relation Evaluation for Large Language Models.","github":"","has_recent":0,"id":"core","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CORE","paper":"","placements":[{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":1,"score":53.47,"version":"Nanbeige4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":2,"score":47.52,"version":"Qwen3 VL"}],"stars":"","top_company":"boss","top_model":"Nanbeige4.1-3B","top_model_family":"Nanbeige","top_model_id":"nanbeige4.1-3b","top_model_version":"Nanbeige4.1","top_open_source":1,"top_percent":53.47,"topic":"Ontological reasoning","total_models":1613,"url":""},{"bench_models":11,"description":"Question answering over large text corpora.","github":"","has_recent":0,"id":"corpusqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CorpusQA","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":81.56,"version":"GPT-5"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1.5-30b-a3b","place":2,"score":81.25,"version":"QwenLong"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":80.62,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":4,"score":79.38,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":5,"score":77.5,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":6,"score":75.31,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking-preview","place":7,"score":74.69,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":8,"score":71.56,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":9,"score":64.38,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":10,"score":50.31,"version":"GPT-5"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1","place":11,"score":42.5,"version":"QwenLong"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":81.56,"topic":"Long-context QA","total_models":1613,"url":""},{"bench_models":56,"description":"Object counting and numeracy benchmark for visual-language models across varied scenes.","github":"","has_recent":0,"id":"countbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CountBench","paper":"https://arxiv.org/abs/2302.12066","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":1,"score":97.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":2,"score":97.8,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":97.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":97.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":97,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":6,"score":96.3,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":96.1,"version":"Gemini 3"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":8,"score":95.1,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":9,"score":94.9,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":10,"score":94.1,"version":"Kimi K2.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":11,"score":93.9,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":12,"score":93.7,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":13,"score":93.7,"version":"Seed 1.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":14,"score":93.7,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":15,"score":93.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":16,"score":93.6,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":17,"score":93.1,"version":"Claude 4.1"},{"company":"tencent","family":"R","model":"r-4b-base","place":18,"score":92.6,"version":"R"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":19,"score":92.4,"version":"Qwen2.5 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":20,"score":92.4,"version":"GLM 4.6V"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":21,"score":91.99,"version":"dots.vlm1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":22,"score":91.9,"version":"GPT-5.2"},{"company":"unknown","family":"PLM","model":"plm-8b","place":23,"score":91.8,"version":"PLM"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":24,"score":91.8,"version":"Seed 1.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":25,"score":91.8,"version":"Ovis 2.6"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":26,"score":91.79,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":27,"score":91.7,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":28,"score":91.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":29,"score":91.5,"version":"Claude 4"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":30,"score":91.2,"version":"Molmo"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":31,"score":91.2,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":32,"score":91,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":33,"score":91,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":34,"score":91,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":35,"score":91,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":36,"score":90.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":37,"score":90.8,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":38,"score":90.6,"version":"Claude 4.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":39,"score":90.4,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":40,"score":90.4,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":41,"score":90.2,"version":"Claude 3.7"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":42,"score":90.2,"version":"Eagle 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":43,"score":90,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":44,"score":89.8,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":45,"score":89,"version":"Seed 1.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":46,"score":89,"version":"ERNIE 4.5 VL"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":47,"score":89,"version":"Molmo"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":48,"score":88.5,"version":"Molmo"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":49,"score":88.4,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":50,"score":88,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":51,"score":87.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":52,"score":87.1,"version":"GPT-5"},{"company":"unknown","family":"PLM","model":"plm-3b","place":53,"score":87.1,"version":"PLM"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":54,"score":87,"version":"MiMo VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":55,"score":86.7,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":56,"score":85.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":57,"score":85.7,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":58,"score":85.5,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":59,"score":84.1,"version":"GPT-5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":60,"score":83.9,"version":"MiniCPM V 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":61,"score":82.2,"version":"InternVL 3.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":62,"score":81.6,"version":"Keye-VL 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":63,"score":80,"version":"InternVL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":64,"score":79.6,"version":"InternVL 3.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":65,"score":77.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":66,"score":74.1,"version":"Qwen2.5 VL"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-27B","top_model_family":"Qwen","top_model_id":"qwen3.5-27b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":97.8,"topic":"Visual counting","total_models":1613,"url":"https://huggingface.co/datasets/nielsr/countbench"},{"bench_models":31,"description":"Visual question answering benchmark focused on counting objects across varied scenes.","github":"","has_recent":0,"id":"countbenchqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CountBenchQA","paper":"","placements":[{"company":"moondream","family":"Moondream","model":"moondream-9b-a2b","place":1,"score":93.2,"version":"Moondream 3"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":2,"score":91.2,"version":"Molmo"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":3,"score":90.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":4,"score":89.7,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":5,"score":89.3,"version":"GPT-5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":6,"score":89,"version":"Molmo"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":7,"score":88.5,"version":"Molmo"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":8,"score":87.9,"version":"GPT-4o"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":9,"score":87.2,"version":"MolmoE"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":10,"score":85.8,"version":"Gemini 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":11,"score":84.3,"version":"LLaVA OneVision"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":12,"score":83.6,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":13,"score":83,"version":"Claude 3"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":14,"score":81.9,"version":"xGen-MM"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":15,"score":81.6,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":16,"score":81.2,"version":"Gemini 2.5"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":17,"score":80.6,"version":"PaliGemma"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":18,"score":80.4,"version":"Qwen2 VL"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":19,"score":78.8,"version":"LLaVA OneVision"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":20,"score":78.8,"version":"Pixtral"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":21,"score":78.5,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":22,"score":76.5,"version":"Qwen2 VL"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":23,"score":76.4,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":24,"score":75.6,"version":"Cambrian 1"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":25,"score":74.7,"version":"InternVL 2"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":26,"score":73.1,"version":"Llama 3.2"},{"company":"openai","family":"GPT","model":"gpt-4v","place":27,"score":69.9,"version":"GPT-4V"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":28,"score":64.6,"version":"Phi 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":29,"score":57.8,"version":"InternVL 2"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":30,"score":47.1,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":31,"score":40.1,"version":"LLaVA 1.5"}],"stars":"","top_company":"moondream","top_model":"Moondream-9B-A2B","top_model_family":"Moondream","top_model_id":"moondream-9b-a2b","top_model_version":"Moondream 3","top_open_source":1,"top_percent":93.2,"topic":"Visual counting QA","total_models":1613,"url":""},{"bench_models":5,"description":"Countdown-style reasoning and planning benchmark.","github":"","has_recent":0,"id":"countdown","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Countdown","paper":"","placements":[{"company":"mbzuai","family":"K2","model":"k2-v2","place":1,"score":75.6,"version":"K2 V2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":2,"score":23.2,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":3,"score":6,"version":"Qwen2.5"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":4,"score":1.3,"version":"K2 V2"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":5,"score":1,"version":"Llama 3.0"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":6,"score":0.5,"version":"Llama 3.1"}],"stars":"","top_company":"mbzuai","top_model":"K2-V2","top_model_family":"K2","top_model_id":"k2-v2","top_model_version":"K2 V2","top_open_source":1,"top_percent":75.6,"topic":"Planning and reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Video-based counting benchmark for multiple objects.","github":"","has_recent":0,"id":"countix","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Countix","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":31,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":2,"score":26,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":18.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":18.6,"version":"Gemini 2.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":31,"topic":"Video counting","total_models":1613,"url":""},{"bench_models":4,"description":"Complex Retrieval-Augmented Generation benchmark for grounded question answering.","github":"","has_recent":0,"id":"crag","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CRAG","paper":"","placements":[{"company":"ai21","family":"Jamba","model":"jamba-mini-1.6","place":1,"score":76.2,"version":"Jamba Mini"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":2,"score":60,"version":"Llama 3.1"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":3,"score":52,"version":"Ministral"},{"company":"cohere","family":"Command","model":"command-r7b","place":4,"score":23.1,"version":"Command"}],"stars":"","top_company":"ai21","top_model":"Jamba Mini 1.6","top_model_family":"Jamba","top_model_id":"jamba-mini-1.6","top_model_version":"Jamba Mini","top_open_source":0,"top_percent":76.2,"topic":"Retrieval QA","total_models":1613,"url":""},{"bench_models":17,"description":"Story writing benchmark evaluating creativity, coherence, and style (v3).","github":"https://github.com/lechmazur/writing","has_recent":0,"id":"creative-story-writing-v3","leaderboard":"https://github.com/lechmazur/writing","lower_is_better":false,"metric":"score","name":"Creative Story‑Writing Benchmark V3","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":1,"score":8.749,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":8.723,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":3,"score":8.711,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":4,"score":8.693,"version":"Kimi K2"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3.1","place":5,"score":8.629,"version":"Mistral 3.1"},{"company":"openai","family":"o3","model":"o3-pro","place":6,"score":8.628,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":8.602,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":8,"score":8.538,"version":"Claude 4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":9,"score":8.517,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":10,"score":8.507,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":11,"score":8.495,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":12,"score":8.11,"version":"ERNIE 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":13,"score":8.068,"version":"Grok 4"},{"company":"zai","family":"GLM","model":"glm-4.5","place":14,"score":7.803,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":15,"score":7.726,"version":"GPT-OSS"},{"company":"cohere","family":"Command","model":"command-a","place":16,"score":7.459,"version":"Command"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":17,"score":6.37,"version":"Llama 4"}],"stars":"291","top_company":"moonshotai","top_model":"Kimi-K2-Instruct-0905","top_model_family":"Kimi","top_model_id":"kimi-k2-instruct-0905","top_model_version":"Kimi K2","top_open_source":1,"top_percent":8.749,"topic":"Creative writing","total_models":1613,"url":""},{"bench_models":16,"description":"Longform creative writing evaluation (EQ-Bench).","github":"https://github.com/EQ-bench","has_recent":0,"id":"creative-writing-longform","leaderboard":"https://eqbench.com/creative_writing_longform.html","lower_is_better":false,"metric":"percent","name":"Longform Creative Writing","paper":"https://arxiv.org/abs/2312.06281","placements":[{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":1,"score":79.8,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0528","place":2,"score":78.9,"version":"DeepSeek V3"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t2-chimera","place":3,"score":76.4,"version":"DeepSeek-TNG R1T2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":4,"score":74.1,"version":"Claude 4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":5,"score":73.8,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":6,"score":73.6,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":7,"score":73.5,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":8,"score":72.5,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":72,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":10,"score":71.6,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-5","place":11,"score":71.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":12,"score":70.9,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":13,"score":70.7,"version":"DeepSeek R1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":14,"score":69.5,"version":"GLM 4.5"},{"company":"nousresearch","family":"Hermes","model":"hermes-4-405b","place":15,"score":66.8,"version":"Hermes 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":16,"score":62.8,"version":"Kimi K2"}],"stars":"20","top_company":"anthropic","top_model":"Claude Sonnet 4.5 (Thinking)","top_model_family":"Claude","top_model_id":"claude-4.5-sonnet-thinking","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":79.8,"topic":"Creative writing","total_models":1613,"url":"https://eqbench.com/creative_writing_longform.html"},{"bench_models":61,"description":"A LLM-judged creative writing benchmark.","github":"https://github.com/EQ-bench","has_recent":0,"id":"creative-writing-v3","leaderboard":"https://eqbench.com/creative_writing.html","lower_is_better":false,"metric":"ELO","name":"Creative Writing v3","paper":"https://arxiv.org/abs/2312.06281","placements":[{"company":"openai","family":"o3","model":"o3","place":1,"score":1661,"version":"o3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":2,"score":1648.4,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":3,"score":1631,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":1628.3,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":5,"score":1500,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":6,"score":1499.9,"version":"Claude 4"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3.1","place":7,"score":1489,"version":"Mistral 3.1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":8,"score":1480.2,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":9,"score":1477.9,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":10,"score":1444.3,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking-high","place":11,"score":89.69,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":12,"score":87.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":13,"score":86,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":14,"score":86,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":15,"score":85.7,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":16,"score":85.5,"version":"Ring"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":17,"score":85.49,"version":"Qwen3"},{"company":"antgroup","family":"Ring","model":"ring-1t","place":18,"score":85.4,"version":"Ring"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":19,"score":85.3,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":20,"score":85.24,"version":"DeepSeek V3.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":21,"score":85.17,"version":"Ling 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":22,"score":85,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":23,"score":84.9,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":24,"score":84.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":25,"score":84.6,"version":"Qwen3 VL"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":26,"score":84.51,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":27,"score":84.4,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":28,"score":84.33,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":29,"score":83.6,"version":"Qwen3"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":30,"score":83.58,"version":"Ring 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":31,"score":83.54,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":32,"score":83.46,"version":"Seed OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":33,"score":82.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":34,"score":82.5,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":35,"score":82.17,"version":"Seed OSS"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":36,"score":82.1,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":37,"score":81.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":38,"score":81,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":39,"score":81,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":40,"score":80.77,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":41,"score":80.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":42,"score":80.4,"version":"Qwen3"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":43,"score":80.3,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":44,"score":79.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":45,"score":79.09,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":46,"score":77.57,"version":"Qwen3"},{"company":"menlo","family":"Jan","model":"jan-v1-2509","place":47,"score":75.73,"version":"Jan v1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":48,"score":72.57,"version":"Qwen3 Next"},{"company":"menlo","family":"Jan","model":"jan-v1","place":49,"score":72.08,"version":"Jan v1"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":50,"score":68.56,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking-2507","place":51,"score":65.74,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":52,"score":51.71,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":53,"score":46.96,"version":"Qwen3"},{"company":"menlo","family":"Jan","model":"jan-v1-edge","place":54,"score":45.31,"version":"Jan v1"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":55,"score":44.22,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":56,"score":41.67,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":57,"score":38.84,"version":"Llama 3.2"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":58,"score":38.79,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":59,"score":37.49,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":60,"score":36.44,"version":"SmollM3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":61,"score":32.6,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":62,"score":31.56,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":63,"score":31.43,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":64,"score":22.18,"version":"Qwen2.5"}],"stars":"54","top_company":"openai","top_model":"o3","top_model_family":"o3","top_model_id":"o3","top_model_version":"o3","top_open_source":0,"top_percent":1661,"topic":"Creative writing","total_models":1613,"url":"https://eqbench.com/creative_writing.html"},{"bench_models":38,"description":"CritPt (Complex Research using Integrated Thinking – Physics Test) benchmark.","github":"","has_recent":0,"id":"critpt","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Complex Research using Integrated Thinking – Physics Test","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high-code-web","place":1,"score":12.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":11.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":3,"score":11.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-high-code","place":4,"score":10.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":9.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":6,"score":8.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":5.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":4.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":9,"score":4.6,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":4.3,"version":"Claude 4.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":11,"score":4.3,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":12,"score":2.9,"version":"DeepSeek V3.2"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":13,"score":2.9,"version":"Grok 4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":14,"score":2.6,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":15,"score":2,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4","place":16,"score":2,"version":"Grok 4"},{"company":"zai","family":"GLM","model":"glm-4.7","place":17,"score":1.7,"version":"GLM 4.7"},{"company":"openai","family":"o3","model":"o3-high","place":18,"score":1.4,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":19,"score":1.4,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":20,"score":1.1,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":21,"score":1.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":22,"score":1.1,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":23,"score":1.1,"version":"Claude 4.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":24,"score":1.1,"version":"Nemotron 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":25,"score":0.9,"version":"MiniMax M2.1"},{"company":"openai","family":"o4","model":"o4-mini-high","place":26,"score":0.6,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":27,"score":0.3,"version":"Claude 4"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":28,"score":0.3,"version":"Apriel 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":29,"score":0,"version":"GPT-5"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":30,"score":0,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":31,"score":0,"version":"GPT-4o"},{"company":"mistral","family":"Mistral","model":"mistral-large-3","place":32,"score":0,"version":"Mistral 3"},{"company":"amazon","family":"Nova","model":"nova-pro","place":33,"score":0,"version":"Nova"},{"company":"amazon","family":"Nova","model":"nova-lite","place":34,"score":0,"version":"Nova"},{"company":"kuaishou","family":"KAT","model":"kat-coder-pro-v1","place":35,"score":0,"version":"KAT-Coder Pro v1"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":36,"score":0,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-2507","place":37,"score":0,"version":"Qwen3"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":38,"score":0,"version":"Mi:dm K 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 (High, Code \u0026 Web)","top_model_family":"GPT","top_model_id":"gpt-5-high-code-web","top_model_version":"GPT-5","top_open_source":0,"top_percent":12.6,"topic":"Reasoning","total_models":1613,"url":"https://critpt.com/"},{"bench_models":74,"description":"Code Reasoning and Understanding eXam – Interactive.","github":"","has_recent":0,"id":"crux-i","leaderboard":"https://crux-eval.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"CRUX-I","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":1,"score":98.8,"version":"Gemini 3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-thinking","place":2,"score":98.5,"version":"IQuest Coder V1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":98.4,"version":"Claude 4.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":4,"score":97.9,"version":"IQuest Coder V1"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":5,"score":96.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":96.2,"version":"Claude 4.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":7,"score":93.5,"version":"IQuest Coder V1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":8,"score":92.2,"version":"Kimi K2"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":9,"score":91.1,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":10,"score":87.1,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":11,"score":86.8,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":12,"score":82.1,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":13,"score":78.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":14,"score":76.9,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09-cot","place":15,"score":75.7,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4o-cot","place":16,"score":75.6,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":17,"score":75.6,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4-0613-cot","place":18,"score":75.5,"version":"GPT-3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":19,"score":74,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-cot","place":20,"score":73.4,"version":"Claude 4"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":21,"score":71.4,"version":"KAT"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":22,"score":70.8,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-4-0613","place":23,"score":69.8,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":24,"score":68.5,"version":"GPT-4"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":25,"score":67.5,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":26,"score":66.9,"version":"Qwen2.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":27,"score":65.6,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-4o","place":28,"score":65.1,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":29,"score":64.2,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":30,"score":63.9,"version":"DeepSeek V3.2"},{"company":"semcoder","family":"SemCoder","model":"semcoder-s-6.7b-cot","place":31,"score":63.1,"version":"SemCoder"},{"company":"semcoder","family":"SemCoder","model":"semcoder-6.7b-cot","place":32,"score":62.5,"version":"SemCoder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":33,"score":62.1,"version":"DeepSeek V3.1"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":34,"score":62,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":35,"score":62,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":36,"score":61,"version":"Kimi Linear"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":37,"score":57.1,"version":"DeepSeek Coder V2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0613-cot","place":38,"score":50.3,"version":"GPT-3.5"},{"company":"codellama","family":"CodeLlama","model":"codellama-34b-cot","place":39,"score":50.1,"version":"CodeLlama"},{"company":"codetulu","family":"CodeTulu","model":"codetulu-2-34b","place":40,"score":49.3,"version":"CodeTulu"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0613","place":41,"score":49,"version":"GPT-3.5"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b","place":42,"score":48.1,"version":"StarCoder2"},{"company":"codellama","family":"CodeLlama","model":"codellama-13b-cot","place":43,"score":47.4,"version":"CodeLlama"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":44,"score":47.2,"version":"CodeLlama-34B"},{"company":"phind","family":"Phind","model":"phind-base","place":45,"score":47.2,"version":"Phind"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-33b","place":46,"score":46.5,"version":"DeepSeek"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-33b","place":47,"score":46.5,"version":"DeepSeek"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":48,"score":45.9,"version":"Moonlight"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-34b","place":49,"score":43.9,"version":"CodeLlama"},{"company":"wizardlm","family":"WizardLM","model":"wizard-34b","place":50,"score":42.7,"version":"WizardLM"},{"company":"codellama","family":"CodeLlama-13B","model":"codellama-13b","place":51,"score":42.5,"version":"CodeLlama-13B"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":52,"score":42.5,"version":"KAT"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-6.7b","place":53,"score":41.9,"version":"DeepSeek"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":54,"score":41.7,"version":"Magicoder-S-DS-6.7B"},{"company":"codellama","family":"CodeLlama","model":"codellama-7b-cot","place":55,"score":40.4,"version":"CodeLlama"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-13b","place":56,"score":39.7,"version":"CodeLlama"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":57,"score":39.3,"version":"Mixtral"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-6.7b","place":58,"score":37.4,"version":"DeepSeek"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-7b","place":59,"score":37.3,"version":"CodeLlama"},{"company":"wizardlm","family":"WizardLM","model":"wizard-13b","place":60,"score":36.5,"version":"WizardLM"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":61,"score":35.9,"version":"CodeLlama-7B"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":62,"score":35,"version":"Mistral"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-7b","place":63,"score":34.6,"version":"StarCoder2"},{"company":"stability","family":"StableCode","model":"stablecode-3b","place":64,"score":33.5,"version":"StableCode"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":65,"score":33,"version":"Kimi Dev"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-3b","place":66,"score":32.7,"version":"StarCoder2"},{"company":"microsoft","family":"Phi","model":"phi-2","place":67,"score":31.6,"version":"Phi"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-16b","place":68,"score":31.3,"version":"StarCoder"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-7b","place":69,"score":29.7,"version":"StarCoderBase"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-1.3b","place":70,"score":27.8,"version":"DeepSeek"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-1.3b","place":71,"score":27.2,"version":"DeepSeek"},{"company":"microsoft","family":"Phi","model":"phi-1.5","place":72,"score":23.2,"version":"Phi"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":73,"score":15.2,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-1","place":74,"score":13.1,"version":"Phi"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro Preview","top_model_family":"Gemini","top_model_id":"gemini-3-pro-preview","top_model_version":"Gemini 3","top_open_source":0,"top_percent":98.8,"topic":"Code reasoning","total_models":1613,"url":""},{"bench_models":101,"description":"Code Reasoning and Understanding eXam – Offline.","github":"","has_recent":0,"id":"crux-o","leaderboard":"https://crux-eval.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"CRUX-O","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-thinking","place":1,"score":99.4,"version":"IQuest Coder V1"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":2,"score":99.1,"version":"Gemini 3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":3,"score":98.9,"version":"IQuest Coder V1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":98,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":5,"score":97.6,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":96.2,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":7,"score":94.2,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":8,"score":90.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":9,"score":89.5,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":10,"score":89.5,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4-0613-cot","place":11,"score":88.2,"version":"GPT-3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":12,"score":87.5,"version":"LLaDA 2.1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":13,"score":87,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":14,"score":86.75,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":15,"score":86.2,"version":"Kimi K2"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":16,"score":85.5,"version":"IQuest Coder V1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":17,"score":85.25,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":18,"score":85.12,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":19,"score":84,"version":"Qwen2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":20,"score":83.5,"version":"Kimi K2"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":21,"score":82.75,"version":"Ling 2.0"},{"company":"openai","family":"GPT","model":"gpt-4","place":22,"score":82,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09-cot","place":23,"score":82,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-cot","place":24,"score":82,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.7","place":25,"score":81.2,"version":"GLM 4.7"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":26,"score":81.1,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":27,"score":80.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":28,"score":79.2,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":29,"score":79.1,"version":"Qwen2.5"},{"company":"semcoder","family":"SemCoder","model":"semcoder-6.7b-cot","place":30,"score":79.1,"version":"SemCoder"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":31,"score":79.1,"version":"MiMo V2"},{"company":"semcoder","family":"SemCoder","model":"semcoder-s-6.7b-cot","place":32,"score":78.3,"version":"SemCoder"},{"company":"openai","family":"GPT","model":"gpt-4-0613-cot","place":33,"score":77.1,"version":"GPT-3"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0613-cot","place":34,"score":76.7,"version":"GPT-3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":35,"score":76.4,"version":"DeepSeek V3.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":36,"score":76.12,"version":"Ling 2.0"},{"company":"openai","family":"GPT","model":"gpt-4o-cot","place":37,"score":76,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":38,"score":74.9,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":39,"score":74.06,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":40,"score":73.75,"version":"LLaDA 2.1"},{"company":"openai","family":"GPT","model":"gpt-4-0613","place":41,"score":73,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":42,"score":72.5,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":43,"score":71.62,"version":"LLaDA 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":44,"score":71.1,"version":"GPT-5.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":45,"score":70.62,"version":"LLaDA 2.1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":46,"score":70,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":47,"score":69.8,"version":"DeepSeek V3"},{"company":"codellama","family":"CodeLlama","model":"codellama-34b-cot","place":48,"score":69.4,"version":"CodeLlama"},{"company":"openai","family":"GPT","model":"gpt-4-0613","place":49,"score":68.7,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":50,"score":68.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":51,"score":67.7,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":52,"score":67.2,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":53,"score":67,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":54,"score":66.6,"version":"Qwen2.5"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":55,"score":66.6,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":56,"score":66,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":57,"score":65.8,"version":"Claude 4"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":58,"score":65.1,"version":"KAT"},{"company":"semcoder","family":"SemCoder","model":"semcoder-6.7b-cot","place":59,"score":64.4,"version":"SemCoder"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":60,"score":64.2,"version":"Kimi Dev"},{"company":"semcoder","family":"SemCoder","model":"semcoder-s-6.7b-cot","place":61,"score":64.1,"version":"SemCoder"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":62,"score":63.1,"version":"Motif 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-33b","place":63,"score":61.8,"version":"DeepSeek"},{"company":"codellama","family":"CodeLlama","model":"codellama-13b-cot","place":64,"score":61.8,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-33b","place":65,"score":61.6,"version":"DeepSeek"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":66,"score":59.9,"version":"Llama 3.1"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b","place":67,"score":59.5,"version":"StarCoder2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0613","place":68,"score":59.3,"version":"GPT-3.5"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0613-cot","place":69,"score":59,"version":"GPT-3.5"},{"company":"codetulu","family":"CodeTulu","model":"codetulu-2-34b","place":70,"score":58.9,"version":"CodeTulu"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":71,"score":57.5,"version":"Magicoder-S-DS-6.7B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":72,"score":56.2,"version":"DeepSeek Coder V2"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":73,"score":55.9,"version":"CodeLlama-34B"},{"company":"codellama","family":"CodeLlama","model":"codellama-7b-cot","place":74,"score":55.4,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-6.7b","place":75,"score":54.8,"version":"DeepSeek"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":76,"score":54,"version":"Mixtral"},{"company":"codellama","family":"CodeLlama-13B","model":"codellama-13b","place":77,"score":53.9,"version":"CodeLlama-13B"},{"company":"wizardlm","family":"WizardLM","model":"wizard-34b","place":78,"score":53.8,"version":"WizardLM"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-34b","place":79,"score":52.9,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-6.7b","place":80,"score":52.8,"version":"DeepSeek"},{"company":"phind","family":"Phind","model":"phind-base","place":81,"score":52.8,"version":"Phind"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-13b","place":82,"score":52.5,"version":"CodeLlama"},{"company":"wizardlm","family":"WizardLM","model":"wizard-13b","place":83,"score":52.4,"version":"WizardLM"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-7b","place":84,"score":52,"version":"StarCoder2"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":85,"score":51.3,"version":"Codestral"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":86,"score":50.25,"version":"Granite 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-33b","place":87,"score":49.9,"version":"DeepSeek"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":88,"score":49.5,"version":"DeepSeek Coder"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0613","place":89,"score":49.4,"version":"GPT-3.5"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-7b","place":90,"score":48.8,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-33b","place":91,"score":48.6,"version":"DeepSeek"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":92,"score":48.6,"version":"Mistral"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":93,"score":48.4,"version":"CodeLlama-7B"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-3b","place":94,"score":48.4,"version":"StarCoder2"},{"company":"meta","family":"CodeLlama","model":"codellama-70b","place":95,"score":47.3,"version":"CodeLlama"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b","place":96,"score":47.1,"version":"StarCoder2"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-16b","place":97,"score":47.1,"version":"StarCoder"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":98,"score":46.9,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-2","place":99,"score":46.6,"version":"Phi"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":100,"score":46.6,"version":"Moonlight"},{"company":"codetulu","family":"CodeTulu","model":"codetulu-2-34b","place":101,"score":45.8,"version":"CodeTulu"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":102,"score":45.5,"version":"Granite 4.0"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-7b","place":103,"score":44.9,"version":"StarCoderBase"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":104,"score":44.4,"version":"Magicoder-S-DS-6.7B"},{"company":"codellama","family":"CodeLlama","model":"codellama-34b-cot","place":105,"score":43.6,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-6.7b","place":106,"score":43.5,"version":"DeepSeek"},{"company":"stability","family":"StableCode","model":"stablecode-3b","place":107,"score":43.5,"version":"StableCode"},{"company":"wizardlm","family":"WizardLM","model":"wizard-34b","place":108,"score":43.4,"version":"WizardLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-1.3b","place":109,"score":43.4,"version":"DeepSeek"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":110,"score":43,"version":"Granite 4.0"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":111,"score":42.4,"version":"CodeLlama-34B"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":112,"score":41.5,"version":"Granite 4.0"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-34b","place":113,"score":41.4,"version":"CodeLlama"},{"company":"wizardlm","family":"WizardLM","model":"wizard-13b","place":114,"score":41.3,"version":"WizardLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-6.7b","place":115,"score":41.2,"version":"DeepSeek"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":116,"score":40.5,"version":"Mixtral"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-1.3b","place":117,"score":40,"version":"DeepSeek"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-13b","place":118,"score":39.8,"version":"CodeLlama"},{"company":"codellama","family":"CodeLlama-13B","model":"codellama-13b","place":119,"score":39.7,"version":"CodeLlama-13B"},{"company":"phind","family":"Phind","model":"phind-base","place":120,"score":39.7,"version":"Phind"},{"company":"microsoft","family":"Phi","model":"phi-1.5","place":121,"score":39.1,"version":"Phi"},{"company":"codellama","family":"CodeLlama","model":"codellama-13b-cot","place":122,"score":36,"version":"CodeLlama"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-7b","place":123,"score":36,"version":"StarCoder2"},{"company":"codellama","family":"CodeLlama","model":"codellama-python-7b","place":124,"score":35.9,"version":"CodeLlama"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":125,"score":34.3,"version":"Mistral"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-16b","place":126,"score":34.2,"version":"StarCoder"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":127,"score":34.2,"version":"CodeLlama-7B"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-3b","place":128,"score":34.2,"version":"StarCoder2"},{"company":"microsoft","family":"Phi","model":"phi-2","place":129,"score":33.5,"version":"Phi"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-7b","place":130,"score":32.2,"version":"StarCoderBase"},{"company":"microsoft","family":"Phi","model":"phi-1","place":131,"score":32,"version":"Phi"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-base-1.3b","place":132,"score":31,"version":"DeepSeek"},{"company":"codellama","family":"CodeLlama","model":"codellama-7b-cot","place":133,"score":29.9,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-instruct-1.3b","place":134,"score":28.7,"version":"DeepSeek"},{"company":"microsoft","family":"Phi","model":"phi-1.5","place":135,"score":27.5,"version":"Phi"},{"company":"stability","family":"StableCode","model":"stablecode-3b","place":136,"score":26.7,"version":"StableCode"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":137,"score":26,"version":"Llama 3"},{"company":"microsoft","family":"Phi","model":"phi-1","place":138,"score":21.7,"version":"Phi"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Loop-Thinking","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-loop-thinking","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":99.4,"topic":"Code reasoning","total_models":1613,"url":""},{"bench_models":40,"description":"Mathematical coding challenge set from the CruxEval benchmark.","github":"https://github.com/crux-eval/cruxeval","has_recent":0,"id":"cruxeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CruxEval","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":1,"score":86.75,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":2,"score":85.12,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":3,"score":82.75,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":4,"score":78.45,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":5,"score":75.63,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":6,"score":74.82,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":7,"score":73.5,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":8,"score":73.07,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":9,"score":72.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":10,"score":69.63,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":11,"score":69.53,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":12,"score":68.32,"version":"Llama 4"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":13,"score":67.67,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":14,"score":66.51,"version":"Falcon H1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":15,"score":61.88,"version":"LLaDA 2.0"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":16,"score":59.57,"version":"Falcon 3"},{"company":"mistral","family":"Codestral","model":"codestral-mamba","place":17,"score":57.8,"version":"Codestral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":18,"score":56.9,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":19,"score":56.25,"version":"Falcon H1"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":20,"score":55.5,"version":"DeepSeek Math"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":21,"score":55.2,"version":"CodeLlama-34B"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":22,"score":55,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":23,"score":52.32,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":24,"score":52.13,"version":"Gemma 3"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":25,"score":51.3,"version":"Codestral"},{"company":"google","family":"CodeGemma","model":"codegemma-1.1-7b","place":26,"score":50.4,"version":"CodeGemma"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":27,"score":50.1,"version":"CodeLlama-7B"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":28,"score":44.38,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":29,"score":43.26,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":30,"score":39.57,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":31,"score":34.76,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":32,"score":25.75,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":33,"score":23.63,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":34,"score":21.57,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":35,"score":18.88,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":36,"score":17.75,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":37,"score":15.58,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":38,"score":14.88,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":39,"score":12.7,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":40,"score":0.06,"version":"Llama 3.2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Instruct-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-instruct-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":86.75,"topic":"Code reasoning","total_models":1613,"url":"https://github.com/crux-eval/cruxeval"},{"bench_models":11,"description":"Chinese SimpleQA benchmark variant (short factual questions).","github":"","has_recent":0,"id":"csimpleqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CSimpleQA","paper":"","placements":[{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":1,"score":78.97,"version":"Ling 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":2,"score":77.6,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":76.8,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":4,"score":70.9,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":5,"score":68.37,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":6,"score":68,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":67.77,"version":"GPT-5.2"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":8,"score":64.6,"version":"Ling 1T"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":9,"score":61.5,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":10,"score":53.5,"version":"Kimi Linear"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":11,"score":34.7,"version":"Moonlight"}],"stars":"","top_company":"antgroup","top_model":"Ling 2.5 1T","top_model_family":"Ling","top_model_id":"ling-2.5-1t","top_model_version":"Ling 2.5","top_open_source":1,"top_percent":78.97,"topic":"QA","total_models":1613,"url":""},{"bench_models":5,"description":"Customer support question answering benchmark.","github":"","has_recent":0,"id":"customer-support-qa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Customer Support Q\u0026A","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":69,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":65.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":64.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":63.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":59.4,"version":"Claude 4.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":69,"topic":"Customer support QA","total_models":1613,"url":""},{"bench_models":5,"description":"CUTE aggregate capability score.","github":"","has_recent":0,"id":"cute","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CUTE","paper":"","placements":[{"company":"allenai","family":"Bolmo","model":"bolmo-7b","place":1,"score":78.6,"version":"Bolmo 7B"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b","place":2,"score":56.9,"version":"OLMo 3"},{"company":"alephalpha","family":"TFree","model":"tfree-hat-pretrained-7b-base","place":3,"score":54.2,"version":"TFree Hat"},{"company":"meta","family":"BLT","model":"blt-7b","place":4,"score":52.3,"version":"BLT 7B"},{"company":"hku","family":"EvaByte","model":"evabyte-6.5b","place":5,"score":50.8,"version":"EvaByte 6.5B"}],"stars":"","top_company":"allenai","top_model":"Bolmo 7B","top_model_family":"Bolmo","top_model_id":"bolmo-7b","top_model_version":"Bolmo 7B","top_open_source":1,"top_percent":78.6,"topic":"English characters","total_models":1613,"url":""},{"bench_models":21,"description":"Diverse CV tasks for VLMs.","github":"","has_recent":0,"id":"cv-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CV-Bench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":92,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":2,"score":89.68,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":88,"version":"Seed 1.8"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":4,"score":87.3,"version":"GLM 4.5V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":85.9,"version":"Gemini 2.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":6,"score":85.65,"version":"dots.vlm1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":85.36,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":8,"score":85,"version":"GLM 4.1V"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":9,"score":85,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":10,"score":84.9,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":84.8,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":84.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":13,"score":84.15,"version":"Qwen2.5 VL"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":14,"score":83.8,"version":"ERNIE 4.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":15,"score":82.3,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":16,"score":81.8,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":17,"score":81,"version":"InternVL"},{"company":"stepfun","family":"Step","model":"step-3","place":18,"score":80.9,"version":"Step 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":19,"score":79.3,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":20,"score":79.1,"version":"Kimi VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":21,"score":76,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":22,"score":75.4,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":23,"score":75.4,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":24,"score":74.6,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":25,"score":70.4,"version":"Gemma 3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":92,"topic":"Computer vision QA","total_models":1613,"url":""},{"bench_models":9,"description":"CVTG-2K CLIPScore for text rendering in image generation.","github":"","has_recent":0,"id":"cvtg-2k-clipscore","leaderboard":"","lower_is_better":false,"metric":"score","name":"CVTG-2K CLIPScore","paper":"","placements":[{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":1,"score":0.8069,"version":"Seedream"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":2,"score":0.8048,"version":"Z-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":3,"score":0.8017,"version":"Qwen-Image"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":4,"score":0.7982,"version":"GPT Image 1"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":5,"score":0.7975,"version":"Seedream"},{"company":"zai","family":"Z-Image","model":"z-image","place":6,"score":0.7969,"version":"Z-Image"},{"company":"zai","family":"GLM","model":"glm-image","place":7,"score":0.7877,"version":"GLM-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":8,"score":0.7819,"version":"Qwen-Image"},{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":9,"score":0.7372,"version":"Nano Banana"}],"stars":"","top_company":"bytedance","top_model":"Seedream 4.5","top_model_family":"Seedream","top_model_id":"seedream-4.5","top_model_version":"Seedream","top_open_source":0,"top_percent":0.8069,"topic":"Text rendering","total_models":1613,"url":""},{"bench_models":9,"description":"CVTG-2K normalized edit distance (NED) for text rendering.","github":"","has_recent":0,"id":"cvtg-2k-ned","leaderboard":"","lower_is_better":false,"metric":"score","name":"CVTG-2K NED","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-image","place":1,"score":0.9557,"version":"GLM-Image"},{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":2,"score":0.9483,"version":"Seedream"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":3,"score":0.9478,"version":"GPT Image 1"},{"company":"zai","family":"Z-Image","model":"z-image","place":4,"score":0.9367,"version":"Z-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":5,"score":0.929,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":6,"score":0.9281,"version":"Z-Image"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":7,"score":0.9224,"version":"Seedream"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":8,"score":0.9116,"version":"Qwen-Image"},{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":9,"score":0.8754,"version":"Nano Banana"}],"stars":"","top_company":"zai","top_model":"GLM-Image","top_model_family":"GLM","top_model_id":"glm-image","top_model_version":"GLM-Image","top_open_source":1,"top_percent":0.9557,"topic":"Text rendering","total_models":1613,"url":""},{"bench_models":9,"description":"CVTG-2K word accuracy for text rendering in images.","github":"","has_recent":0,"id":"cvtg-2k-word-accuracy","leaderboard":"","lower_is_better":false,"metric":"score","name":"CVTG-2K Word Accuracy","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-image","place":1,"score":0.9116,"version":"GLM-Image"},{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":2,"score":0.899,"version":"Seedream"},{"company":"zai","family":"Z-Image","model":"z-image","place":3,"score":0.8671,"version":"Z-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":4,"score":0.8604,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":5,"score":0.8585,"version":"Z-Image"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":6,"score":0.8569,"version":"GPT Image 1"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":7,"score":0.8451,"version":"Seedream"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":8,"score":0.8288,"version":"Qwen-Image"},{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":9,"score":0.7788,"version":"Nano Banana"}],"stars":"","top_company":"zai","top_model":"GLM-Image","top_model_family":"GLM","top_model_id":"glm-image","top_model_version":"GLM-Image","top_open_source":1,"top_percent":0.9116,"topic":"Text rendering","total_models":1613,"url":""},{"bench_models":3,"description":"Framework with 40 professional-level CTF tasks evaluating LLMs' practical cybersecurity capabilities.","github":"","has_recent":0,"id":"cybench","leaderboard":"https://cybench.github.io/","lower_is_better":true,"metric":"percent","name":"CyBench","paper":"","placements":[{"company":"openai","family":"o3","model":"o3-mini","place":1,"score":22.5,"version":"o3"},{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":2,"score":39,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":3,"score":43,"version":"Grok 4"}],"stars":"","top_company":"openai","top_model":"o3 mini","top_model_family":"o3","top_model_id":"o3-mini","top_model_version":"o3","top_open_source":0,"top_percent":22.5,"topic":"Cybersecurity CTF","total_models":1613,"url":"https://cybench.github.io/"},{"bench_models":4,"description":"Benchmark for cybersecurity-related coding and reasoning tasks.","github":"","has_recent":0,"id":"cybergym","leaderboard":"","lower_is_better":false,"metric":"percent","name":"CyberGym","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":1,"score":50.6,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":2,"score":41.3,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":39.9,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":17.3,"version":"DeepSeek V3.2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5 Thinking","top_model_family":"Claude","top_model_id":"claude-opus-4.5-thinking","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":50.6,"topic":"Cybersecurity tasks","total_models":1613,"url":""},{"bench_models":3,"description":"Capture-the-flag challenge benchmark evaluating cybersecurity problem-solving skills.","github":"","has_recent":0,"id":"cybersecurity-capture-the-flag-challenges","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Cybersecurity Capture The Flag Challenges","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":1,"score":77.6,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":67.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":3,"score":67.4,"version":"GPT-5.2 Codex"}],"stars":"","top_company":"openai","top_model":"GPT-5.3 Codex","top_model_family":"GPT","top_model_id":"gpt-5.3-codex","top_model_version":"GPT-5.3 Codex","top_open_source":0,"top_percent":77.6,"topic":"Cybersecurity CTF","total_models":1613,"url":""},{"bench_models":3,"description":"Cybersecurity Capture The Flag challenges benchmark.","github":"","has_recent":0,"id":"cybersecurity-ctf","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Cybersecurity CTF","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":1,"score":77.6,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":67.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":3,"score":67.4,"version":"GPT-5.2 Codex"}],"stars":"","top_company":"openai","top_model":"GPT-5.3 Codex","top_model_family":"GPT","top_model_id":"gpt-5.3-codex","top_model_version":"GPT-5.3 Codex","top_open_source":0,"top_percent":77.6,"topic":"Cybersecurity CTF","total_models":1613,"url":""},{"bench_models":6,"description":"2D/3D spatial reasoning benchmark.","github":"","has_recent":0,"id":"da-2k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DA-2K","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":1,"score":85.3,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":82.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":78.6,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":76.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":68.2,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":6,"score":25.8,"version":"Seed 1.8"}],"stars":"","top_company":"bytedance","top_model":"Seed1.5-VL-Thinking","top_model_family":"Seed","top_model_id":"seed1.5-vl-thinking","top_model_version":"Seed 1.5","top_open_source":0,"top_percent":85.3,"topic":"Spatial reasoning","total_models":1613,"url":""},{"bench_models":13,"description":"Benchmark evaluating deep planning and multi-step reasoning capabilities.","github":"","has_recent":0,"id":"deep-planning","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Deep Planning","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":1,"score":44.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":44.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":34.3,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":33.9,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":5,"score":28.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":6,"score":24.1,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":23.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":8,"score":22.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":9,"score":22.6,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":10,"score":21.6,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":11,"score":17.9,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":12,"score":17.1,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":13,"score":14.5,"version":"Kimi K2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 Thinking","top_model_family":"GPT","top_model_id":"gpt-5.2-thinking","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":44.6,"topic":"Planning and reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Agentic consulting and writing benchmark.","github":"","has_recent":0,"id":"deepconsult","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DeepConsult","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":57.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":48,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":41,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":38.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":21.8,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":57.2,"topic":"Agentic writing","total_models":1613,"url":""},{"bench_models":4,"description":"Synthetic math problem sets from DeepMind covering arithmetic, algebra, calculus, and more.","github":"https://github.com/deepmind/mathematics_dataset","has_recent":0,"id":"deepmind-math","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DeepMind Mathematics","paper":"https://arxiv.org/abs/1904.01557","placements":[{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":1,"score":59.33,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":2,"score":46.44,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":3,"score":46,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":4,"score":44.56,"version":"Granite 4.0"}],"stars":"","top_company":"IBM","top_model":"Granite-4.0-H-Small","top_model_family":"Granite","top_model_id":"granite-4.0-h-small","top_model_version":"Granite 4.0","top_open_source":1,"top_percent":59.33,"topic":"Math reasoning","total_models":1613,"url":"https://github.com/deepmind/mathematics_dataset"},{"bench_models":5,"description":"Research-oriented agentic writing and planning benchmark.","github":"","has_recent":0,"id":"deepresearchbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DeepResearchBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":49.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":48.7,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":43.5,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":40,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":39.9,"version":"Claude 4.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":49.6,"topic":"Agentic research writing","total_models":1613,"url":""},{"bench_models":5,"description":"Multi-step web search and question answering benchmark.","github":"","has_recent":0,"id":"deepsearchqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DeepSearchQA","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":1,"score":77.1,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":2,"score":76.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":3,"score":71.3,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":63.2,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":5,"score":60.9,"version":"DeepSeek V3.2"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2.5 Thinking","top_model_family":"Kimi","top_model_id":"kimi-k2.5-thinking","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":77.1,"topic":"Deep web search QA","total_models":1613,"url":""},{"bench_models":9,"description":"Dense retrieval and reasoning benchmark for long-context evaluation.","github":"","has_recent":0,"id":"der2-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DeR2 Bench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":69,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":66.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":66,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":60.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":58.9,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":6,"score":58.2,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":7,"score":57.3,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":8,"score":50.3,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":46.6,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":69,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":11,"description":"Translating UI designs into code.","github":"","has_recent":0,"id":"design2code","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Design2Code","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":1,"score":93.4,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":92.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":89.2,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":4,"score":88.6,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":5,"score":88.5,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":6,"score":82.2,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":7,"score":69.8,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":8,"score":64.7,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":9,"score":38.8,"version":"Kimi VL"},{"company":"stepfun","family":"Step","model":"step-3","place":10,"score":34.1,"version":"Step 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":11,"score":16.1,"version":"Gemma 3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-235B-A22B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-235b-a22b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":93.4,"topic":"Coding (UI)","total_models":1613,"url":""},{"bench_models":77,"description":"Leaderboard tracking generative design systems across layout, branding, and marketing tasks.","github":"","has_recent":0,"id":"designarena","leaderboard":"https://designarena.ai","lower_is_better":false,"metric":"elo","name":"DesignArena","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":1,"score":1410,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":2,"score":1363,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":3,"score":1341,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.1-opus-thinking","place":4,"score":1337,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":5,"score":1337,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":6,"score":1329,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":7,"score":1318,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":8,"score":1315,"version":"Claude 3.7"},{"company":"zai","family":"GLM","model":"glm-4.5","place":9,"score":1307,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":10,"score":1306,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":1301,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":12,"score":1291,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":13,"score":1289,"version":"DeepSeek V3"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":14,"score":1289,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":15,"score":1281,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":16,"score":1279,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":17,"score":1276,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2","place":18,"score":1268,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":19,"score":1265,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-2505","place":20,"score":1263,"version":"Mistral"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":21,"score":1262,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":22,"score":1255,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":23,"score":1253,"version":"GLM 4.6"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":24,"score":1253,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":25,"score":1253,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-09-2025","place":26,"score":1249,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":27,"score":1247,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":28,"score":1240,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4-32b","place":29,"score":1237,"version":"GLM 4"},{"company":"xai","family":"Grok","model":"grok-3","place":30,"score":1236,"version":"Grok 3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3.1","place":31,"score":1233,"version":"Mistral 3.1"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":32,"score":1231,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":33,"score":1228,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":34,"score":1226,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":35,"score":1224,"version":"Kimi K2"},{"company":"vercel","family":"Vercel v0","model":"v0-1.5-md","place":36,"score":1222,"version":"Vercel v0"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":37,"score":1213,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":38,"score":1210,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite-preview-09-2025","place":39,"score":1210,"version":"Gemini 2.5"},{"company":"mistral","family":"Devstral","model":"devstral-medium","place":40,"score":1207,"version":"Devstral"},{"company":"mistral","family":"Magistral","model":"magistral-medium-2506","place":41,"score":1206,"version":"Magistral 2506"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-turbo-preview","place":42,"score":1203,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":43,"score":1192,"version":"GPT-4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":44,"score":1190,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":45,"score":1182,"version":"GPT-4.1"},{"company":"vercel","family":"Vercel v0","model":"v0-1.5-lg","place":46,"score":1181,"version":"Vercel v0"},{"company":"openai","family":"o4","model":"o4-mini","place":47,"score":1181,"version":"o4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":48,"score":1174,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":49,"score":1169,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":50,"score":1166,"version":"Qwen3"},{"company":"mistral","family":"Codestral","model":"codestral-2508","place":51,"score":1164,"version":"Codestral"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":52,"score":1150,"version":"Grok Code"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":53,"score":1144,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":54,"score":1140,"version":"Magistral 1.2"},{"company":"xai","family":"Grok","model":"grok-4-fast-thinking","place":55,"score":1137,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":56,"score":1135,"version":"GPT-OSS"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":57,"score":1135,"version":"Mistral 3.2"},{"company":"tesslate","family":"UIGen X","model":"uigen-x-32b-0707","place":58,"score":1130,"version":"UIGen X"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":59,"score":1119,"version":"Grok 3"},{"company":"cohere","family":"Command","model":"command-a","place":60,"score":1119,"version":"Command"},{"company":"tesslate","family":"WEBGEN","model":"webgen-small","place":61,"score":1119,"version":"WEBGEN"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":62,"score":1107,"version":"Llama 4"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":63,"score":1099,"version":"Magistral 1.2"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.1","place":64,"score":1099,"version":"Magistral 1.1"},{"company":"xai","family":"Grok","model":"grok-2","place":65,"score":1098,"version":"Grok 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":66,"score":1094,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":67,"score":1091,"version":"GPT-4o"},{"company":"mistral","family":"Mistral","model":"mistral-large-2.1-2411","place":68,"score":1088,"version":"Mistral 2.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":69,"score":1073,"version":"Gemini 1.5"},{"company":"tesslate","family":"UIGen X","model":"uigen-x-4b","place":70,"score":1065,"version":"UIGen X"},{"company":"mistral","family":"Codestral","model":"codestral-2501","place":71,"score":1056,"version":"Codestral"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":72,"score":1051,"version":"GPT-OSS"},{"company":"mistral","family":"Devstral","model":"devstral-small-1.1","place":73,"score":1042,"version":"Devstral"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.1","place":74,"score":1039,"version":"Magistral 1.1"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-premier","place":75,"score":1034,"version":"Amazon Nova"},{"company":"meta","family":"Llama","model":"llama-4-scout","place":76,"score":1031,"version":"Llama 1"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-pro-v1.0","place":77,"score":1006,"version":"Amazon Nova"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.5 (Thinking)","top_model_family":"Claude","top_model_id":"claude-4.5-sonnet-thinking","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":1410,"topic":"Generative design","total_models":1613,"url":"https://designarena.ai"},{"bench_models":6,"description":"Evaluates whether LLMs can notice subtle errors and minor inconsistencies in text.","github":"","has_recent":0,"id":"detailbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DetailBench","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-4-maverick","place":1,"score":8.7,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":2,"score":4.3,"version":"Llama 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":3,"score":3.6,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":2.9,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":5,"score":2.2,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":0.7,"version":"GPT-5"}],"stars":"","top_company":"meta","top_model":"Llama 4 Maverick","top_model_family":"Llama","top_model_id":"llama-4-maverick","top_model_version":"Llama 4","top_open_source":1,"top_percent":8.7,"topic":"Spot small mistakes","total_models":1613,"url":"https://huggingface.co/datasets/xeophon/detailbench"},{"bench_models":12,"description":"DiscoX benchmark for agentic writing and reasoning.","github":"","has_recent":0,"id":"discox","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DiscoX","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":1,"score":82,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":2,"score":80.3,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":78.6,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":76.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":5,"score":76.3,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":75.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":75.1,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":8,"score":74.6,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":73.5,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":10,"score":73,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":11,"score":71.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":71.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":70.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":14,"score":67.7,"version":"GPT-5"}],"stars":"","top_company":"bytedance","top_model":"Seed2.0 Pro","top_model_family":"Seed","top_model_id":"seed2.0-pro","top_model_version":"Seed 2.0","top_open_source":0,"top_percent":82,"topic":"Agentic writing","total_models":1613,"url":""},{"bench_models":5,"description":"Resistance to Do Anything Now (DAN) style jailbreak prompts.","github":"","has_recent":0,"id":"do-anything-now","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Do-Anything-Now","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":1,"score":97.7,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":2,"score":95.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":3,"score":69.7,"version":"Qwen2.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":4,"score":63.3,"version":"IQuest Coder V1"},{"company":"mbzuai","family":"K2","model":"k2-think","place":5,"score":43,"version":"K2-THINK"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Thinking","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-thinking","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":97.7,"topic":"Safety / jailbreak","total_models":1613,"url":""},{"bench_models":5,"description":"Evaluates a model's ability to refuse unsafe or disallowed requests.","github":"https://github.com/Libr-AI/do-not-answer","has_recent":0,"id":"do-not-answer","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Do-Not-Answer","paper":"https://aclanthology.org/2024.findings-eacl.61","placements":[{"company":"mbzuai","family":"K2","model":"k2-think","place":1,"score":88,"version":"K2-THINK"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":2,"score":69.9,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":3,"score":58.6,"version":"IQuest Coder V1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":4,"score":53.9,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":5,"score":53.7,"version":"Qwen2.5"}],"stars":"","top_company":"mbzuai","top_model":"K2-THINK","top_model_family":"K2","top_model_id":"k2-think","top_model_version":"K2-THINK","top_open_source":1,"top_percent":88,"topic":"Safety / refusal","total_models":1613,"url":"https://huggingface.co/datasets/LibrAI/do-not-answer"},{"bench_models":43,"description":"Math reasoning on document-based problems.","github":"","has_recent":0,"id":"docmath","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DocMath","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":67.62,"version":"GPT-5"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1.5-30b-a3b","place":2,"score":66.26,"version":"QwenLong"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":3,"score":65.75,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":4,"score":64.75,"version":"Gemini 2.5"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1","place":5,"score":64.75,"version":"QwenLong"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking-preview","place":6,"score":64.12,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":7,"score":63.88,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":8,"score":63.44,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":9,"score":62.4,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":62.38,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":11,"score":62.26,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":12,"score":61.9,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":13,"score":61.8,"version":"Claude 3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":14,"score":61.25,"version":"GPT-OSS"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":15,"score":59.7,"version":"Mistral"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":16,"score":59.2,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet","place":17,"score":58.5,"version":"Claude 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2","place":18,"score":57.3,"version":"DeepSeek Coder"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":19,"score":54.5,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":20,"score":54.1,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":21,"score":52.8,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":22,"score":52.5,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen2-72b","place":23,"score":52.4,"version":"Qwen2"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":24,"score":52.2,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":25,"score":50.1,"version":"Llama 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":26,"score":47.6,"version":"Mixtral"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":27,"score":41.8,"version":"Gemma 2"},{"company":"cohere","family":"Command","model":"command-r-plus","place":28,"score":39.9,"version":"Command"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-9b","place":29,"score":38.1,"version":"Yi 1.5"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-34b","place":30,"score":36.9,"version":"Yi 1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":31,"score":35.1,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo","place":32,"score":34,"version":"GPT-3.5"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":33,"score":31,"version":"Codestral"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":34,"score":30.1,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen2-7b","place":35,"score":29.9,"version":"Qwen2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":36,"score":21.9,"version":"Mixtral"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":37,"score":21.8,"version":"DeepSeek Math"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":38,"score":20.8,"version":"Llama 2"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b","place":39,"score":11.5,"version":"StarCoder2"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":40,"score":9,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":41,"score":8.4,"version":"Llama 2"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":42,"score":7.6,"version":"DeepSeek Coder"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":43,"score":4.1,"version":"Gemma 2"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":67.62,"topic":"Document math","total_models":1613,"url":""},{"bench_models":113,"description":"Visual question answering over scanned documents.","github":"","has_recent":0,"id":"docvqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DocVQA","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":1,"score":96.9,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":2,"score":96.9,"version":"Qwen3 VL"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":3,"score":96.7,"version":"Ovis 2.6"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":4,"score":96.52,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":5,"score":96.5,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":6,"score":96.5,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":7,"score":96.5,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":8,"score":96.4,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":9,"score":96.23,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":10,"score":96.1,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":11,"score":95.7,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":12,"score":95.5,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":95.42,"version":"Gemini 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":14,"score":95.3,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":15,"score":95.3,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":16,"score":95.2,"version":"Claude 3.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":17,"score":95.2,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":18,"score":95,"version":"Qwen3 VL"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":19,"score":94.86,"version":"Mistral 3.2"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":20,"score":94.7,"version":"MiniCPM V 4.5"},{"company":"unknown","family":"PLM","model":"plm-8b","place":21,"score":94.6,"version":"PLM"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":22,"score":94.5,"version":"Qwen2 VL"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":23,"score":94.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":24,"score":94.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":25,"score":94.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":26,"score":94.4,"version":"Llama 4"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":27,"score":94.3,"version":"Manzano"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":28,"score":94.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":29,"score":94.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":30,"score":94.1,"version":"Claude 3.7"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":31,"score":94.1,"version":"InternVL 2"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":32,"score":94.1,"version":"Eagle 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":33,"score":94.08,"version":"Mistral 3.1"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":34,"score":94,"version":"InternVL 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":35,"score":94,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":36,"score":93.9,"version":"Qwen2.5 VL"},{"company":"unknown","family":"PLM","model":"plm-3b","place":37,"score":93.8,"version":"PLM"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":38,"score":93.65,"version":"Seed 1.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":39,"score":93.6,"version":"ERNIE 4.5 VL"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":40,"score":93.5,"version":"Molmo"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":41,"score":93.5,"version":"Manzano"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":42,"score":93.4,"version":"Keye-VL 1.5"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":43,"score":93.3,"version":"Pixtral"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":44,"score":93.3,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":45,"score":93.3,"version":"GLM 4.1V"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":46,"score":93.2,"version":"FastVLM"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":47,"score":93.2,"version":"Molmo2"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":48,"score":93.1,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":49,"score":93,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":50,"score":92.8,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o","place":51,"score":92.8,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":52,"score":92.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":53,"score":92.5,"version":"Claude 4.1"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":54,"score":92.4,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":55,"score":92.3,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":56,"score":92.3,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":57,"score":92.2,"version":"Molmo"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":58,"score":92.2,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":59,"score":92,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":60,"score":91.7,"version":"Claude 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":61,"score":91.6,"version":"InternVL 2"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-base","place":62,"score":91.6,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":63,"score":91.6,"version":"Llama 4"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":64,"score":91.6,"version":"InternVL 2.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":65,"score":91.6,"version":"LLaVA OneVision"},{"company":"openai","family":"GPT","model":"gpt-5","place":66,"score":91.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":67,"score":91.5,"version":"Gemini 2.5"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":68,"score":91.4,"version":"MM1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":69,"score":91.3,"version":"LLaVA OneVision"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":70,"score":91.2,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":71,"score":91.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":72,"score":91,"version":"GPT-5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":73,"score":90.8,"version":"Molmo"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":74,"score":90.7,"version":"Pixtral"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":75,"score":90.6,"version":"jina-VLM"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":76,"score":90.4,"version":"Molmo2"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":77,"score":90.3,"version":"Claude 3.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":78,"score":90.1,"version":"Llama 3.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":79,"score":90,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":80,"score":89.9,"version":"Gemini 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":81,"score":89.4,"version":"InternVL"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-base","place":82,"score":89.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-base","place":83,"score":89.4,"version":"Llama 4"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":84,"score":89.4,"version":"InternVL 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":85,"score":89.3,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":86,"score":89.2,"version":"Qwen2 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":87,"score":88.9,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-5","place":88,"score":88.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":89,"score":88.8,"version":"Claude 3"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":90,"score":88.7,"version":"InternVL 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":91,"score":88.6,"version":"Qwen2.5 Omni"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":92,"score":88.5,"version":"InternVL 3.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":93,"score":88.4,"version":"Llama 3.2"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":94,"score":88.3,"version":"FastVLM"},{"company":"moondream","family":"Moondream","model":"moondream-9b-a2b","place":95,"score":88.3,"version":"Moondream 3"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":96,"score":88.1,"version":"Bagel"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":97,"score":87.8,"version":"Molmo2"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":98,"score":87.7,"version":"MM1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":99,"score":87.5,"version":"LLaVA OneVision"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":100,"score":87.4,"version":"InternVL 3"},{"company":"openai","family":"GPT","model":"gpt-4v","place":101,"score":87.2,"version":"GPT-4V"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":102,"score":87.2,"version":"MolmoE"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":103,"score":87.1,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":104,"score":86.7,"version":"GPT-5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":105,"score":86.6,"version":"Gemma 3"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":106,"score":83.3,"version":"Phi-3"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":107,"score":82.5,"version":"FastVLM"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-0827","place":108,"score":79.5,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":109,"score":78.3,"version":"GPT-5"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":110,"score":77.8,"version":"Cambrian 1"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":111,"score":77.7,"version":"MolmoE"},{"company":"baai","family":"Emu","model":"emu3-8b","place":112,"score":76.3,"version":"Emu3"},{"company":"apple","family":"MM1","model":"mm1-30b","place":113,"score":75.8,"version":"MM1"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":114,"score":75.5,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":115,"score":75.5,"version":"Cambrian"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":116,"score":74.6,"version":"Claude 3"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":117,"score":74.3,"version":"Gemini Nano"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":118,"score":71.9,"version":"MiniCPM V 2.0"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":119,"score":69.3,"version":"Phi 3.5"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":120,"score":61.4,"version":"xGen-MM"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":121,"score":54,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":122,"score":54,"version":"Claude 4"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":123,"score":40.8,"version":"Janus Pro"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":124,"score":35.5,"version":"Janus Pro"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":125,"score":31.3,"version":"PaliGemma"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":126,"score":30.3,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":127,"score":28.1,"version":"LLaVA 1.5"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":128,"score":0,"version":"VILA 1.5"},{"company":"bytedance","family":"LLaVA","model":"llava-next-34b","place":129,"score":0,"version":"LLaVA-NeXT"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":130,"score":0,"version":"BLIP-3O"}],"stars":"","top_company":"bytedance","top_model":"Seed1.5-VL-Thinking","top_model_family":"Seed","top_model_id":"seed1.5-vl-thinking","top_model_version":"Seed 1.5","top_open_source":0,"top_percent":96.9,"topic":"Document understanding (VQA)","total_models":1613,"url":""},{"bench_models":2,"description":"Dolphin Page benchmark measuring OCR fidelity and structured extraction on multi-layout documents.","github":"","has_recent":0,"id":"dolphin-page","leaderboard":"","lower_is_better":true,"metric":"percent","name":"Dolphin-Page","paper":"","placements":[{"company":"bytedance","family":"Dolphin","model":"dolphin-1.5","place":1,"score":7.43,"version":"Dolphin"},{"company":"bytedance","family":"Dolphin","model":"dolphin","place":2,"score":10.28,"version":"Dolphin"}],"stars":"","top_company":"bytedance","top_model":"Dolphin 1.5","top_model_family":"Dolphin","top_model_id":"dolphin-1.5","top_model_version":"Dolphin","top_open_source":1,"top_percent":7.43,"topic":"Document OCR","total_models":1613,"url":""},{"bench_models":14,"description":"DPG-Bench score for text rendering in image generation.","github":"","has_recent":0,"id":"dpg-bench","leaderboard":"","lower_is_better":false,"metric":"score","name":"DPG-Bench","paper":"","placements":[{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":1,"score":88.63,"version":"Seedream"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":2,"score":88.54,"version":"Seedream"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":3,"score":88.32,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image","place":4,"score":88.14,"version":"Z-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":5,"score":87.2,"version":"Qwen-Image"},{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":6,"score":87.16,"version":"Nano Banana"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":7,"score":85.15,"version":"GPT Image 1"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":8,"score":84.86,"version":"Z-Image"},{"company":"zai","family":"GLM","model":"glm-image","place":9,"score":84.78,"version":"GLM-Image"},{"company":"bytedance","family":"Janus","model":"janus-pro","place":10,"score":84.19,"version":"Janus Pro"},{"company":"stabilityai","family":"Stable Diffusion 3","model":"sd3-medium","place":11,"score":84.08,"version":"SD3"},{"company":"black-forest-labs","family":"FLUX.1","model":"flux-1-dev","place":12,"score":83.52,"version":"FLUX.1"},{"company":"openai","family":"DALL-E","model":"dall-e-3","place":13,"score":83.5,"version":"DALL-E 3"},{"company":"stabilityai","family":"Stable Diffusion","model":"sd-xl","place":14,"score":74.65,"version":"SD XL"}],"stars":"","top_company":"bytedance","top_model":"Seedream 4.5","top_model_family":"Seedream","top_model_id":"seedream-4.5","top_model_version":"Seedream","top_open_source":0,"top_percent":88.63,"topic":"Text rendering","total_models":1613,"url":""},{"bench_models":68,"description":"Discrete reasoning over paragraphs (addition, counting, comparisons).","github":"","has_recent":0,"id":"drop","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DROP","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":1,"score":93.5,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":2,"score":92.2,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":3,"score":92,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":4,"score":91.2,"version":"DeepSeek V3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":5,"score":91.1,"version":"ERNIE 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":6,"score":89.7,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":7,"score":89.2,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":8,"score":88.7,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":9,"score":88.32,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":10,"score":87.9,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":11,"score":87.86,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":12,"score":87.57,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":13,"score":87.55,"version":"LLaDA 2.1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":14,"score":87.2,"version":"Claude 3.7"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":15,"score":86.7,"version":"Youtu-LLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":16,"score":86.6,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":17,"score":86.3,"version":"DeepSeek V3.1"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":18,"score":85.1,"version":"MiMo VL"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":19,"score":84.8,"version":"Llama 3.1"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":20,"score":84.7,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":21,"score":84.56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":22,"score":84.3,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":23,"score":83.6,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":24,"score":82.9,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":25,"score":82.7,"version":"MiMo VL"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":26,"score":82.37,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":27,"score":81.89,"version":"LLaDA 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":28,"score":81.7,"version":"Gemini 2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":29,"score":81.55,"version":"LLaDA 2.1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":30,"score":81.5,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":31,"score":79.7,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":32,"score":79.6,"version":"Llama 3.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":33,"score":79.49,"version":"LLaDA 2.0"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":34,"score":79.1,"version":"GPT-4.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":35,"score":78.8,"version":"Ling 2.0"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":36,"score":77.5,"version":"InternVL"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":37,"score":77.2,"version":"Gemma 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":38,"score":77.2,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":39,"score":77,"version":"Llama 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":40,"score":74.3,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":41,"score":74.1,"version":"Tulu 3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":42,"score":73.2,"version":"Hermes 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":43,"score":73.1,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":44,"score":72.5,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":45,"score":72.2,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":46,"score":72.2,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":47,"score":72,"version":"SmollM3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":48,"score":69.9,"version":"Motif 2"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":49,"score":68.8,"version":"Llama 3.1 Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":50,"score":67.6,"version":"Qwen2.5 VL"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":51,"score":62.6,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":52,"score":62.5,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":53,"score":61.5,"version":"Llama 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":54,"score":61.3,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":55,"score":59.5,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":56,"score":59.36,"version":"Granite 3.3"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":57,"score":58.8,"version":"Gemma 2"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":58,"score":58.57,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":59,"score":58.4,"version":"Llama 3"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":60,"score":58.29,"version":"Granite 3.2"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":61,"score":56.2,"version":"Ministral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":62,"score":54.4,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":63,"score":51.78,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":64,"score":49.73,"version":"DeepSeek R1"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":65,"score":49.4,"version":"Magpie"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":66,"score":46.22,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":67,"score":44.33,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":68,"score":41.53,"version":"Granite 3.3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":69,"score":41.3,"version":"DeepSeek R1"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":70,"score":35.98,"version":"Granite 3.3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":71,"score":34.2,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":72,"score":23.84,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":73,"score":20.99,"version":"Granite 3.1"}],"stars":"","top_company":"moonshotai","top_model":"Kimi K2 Instruct","top_model_family":"Kimi","top_model_id":"kimi-k2-instruct","top_model_version":"Kimi K2","top_open_source":1,"top_percent":93.5,"topic":"Reading + reasoning","total_models":1613,"url":"https://allenai.org/data/drop"},{"bench_models":6,"description":"Long-context multimodal understanding benchmark.","github":"","has_recent":0,"id":"dude","leaderboard":"","lower_is_better":false,"metric":"percent","name":"DUDE","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":70.1,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":69.4,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":67.3,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":67.1,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":65.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":6,"score":54,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":70.1,"topic":"Multimodal long-context","total_models":1613,"url":""},{"bench_models":30,"description":"Dynamic/video-based mathematical reasoning evaluating temporal and visual understanding.","github":"https://github.com/DynaMath/DynaMath","has_recent":0,"id":"dynamath","leaderboard":"https://dynamath.github.io/","lower_is_better":false,"metric":"percent","name":"DynaMath","paper":"https://arxiv.org/pdf/2411.00836","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":1,"score":87.7,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":86.8,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":86.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":85.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":85.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":85,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":84.4,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":8,"score":82.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":9,"score":81.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":79.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":78.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":12,"score":63.7,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":63.3,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":61.5,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":15,"score":61.5,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":16,"score":57.1,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":17,"score":56.3,"version":"Gemini 2.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":18,"score":55.5,"version":"Ovis 2.6"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":19,"score":54.5,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":20,"score":53.9,"version":"GLM 4.5V"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":21,"score":52.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":22,"score":52.5,"version":"GPT-5"},{"company":"stepfun","family":"Step","model":"step-3","place":23,"score":50.1,"version":"Step 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":24,"score":48.7,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":25,"score":43.7,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":26,"score":42.5,"version":"GLM 4.1V"},{"company":"tencent","family":"R","model":"r-4b-base","place":27,"score":39.5,"version":"R"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":28,"score":35.7,"version":"InternVL 3.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":29,"score":35.3,"version":"Keye-VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":30,"score":28.5,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":31,"score":28.1,"version":"Kimi VL"},{"company":"opengvlab","family":"Intern","model":"internvl3-8b","place":32,"score":23.9,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":33,"score":20.1,"version":"Qwen2.5 VL"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-27B","top_model_family":"Qwen","top_model_id":"qwen3.5-27b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":87.7,"topic":"Math reasoning (video)","total_models":1613,"url":"https://dynamath.github.io/"},{"bench_models":2,"description":"Evaluation suite of real-world, economically impactful tasks across key industries and workflows.","github":"","has_recent":0,"id":"economically-important-tasks","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Economically important tasks","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":47.1,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":33.5,"version":"o3"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":47.1,"topic":"Industry QA (cross-domain)","total_models":1613,"url":""},{"bench_models":5,"description":"Education field evaluation (economically valuable tasks).","github":"","has_recent":0,"id":"education-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Education","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":60.8,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":57,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":55,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":53,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":52.4,"version":"Gemini 2.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":60.8,"topic":"Economics/education","total_models":1613,"url":""},{"bench_models":13,"description":"EgoSchema validation accuracy.","github":"","has_recent":0,"id":"egoschema","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EgoSchema","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b-instruct","place":1,"score":77.9,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":2,"score":76.2,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":3,"score":72.2,"version":"GPT-4o"},{"company":"amazon","family":"Nova","model":"nova-pro","place":4,"score":72.1,"version":"Nova"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":5,"score":71.5,"version":"Gemini 2.0"},{"company":"amazon","family":"Nova","model":"nova-lite","place":6,"score":71.4,"version":"Nova"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":7,"score":68.6,"version":"Qwen2.5 Omni"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":8,"score":68.2,"version":"InternVL"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-lite-preview","place":9,"score":67.2,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":10,"score":62.4,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":11,"score":60.4,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":12,"score":59.6,"version":"MiMo VL"},{"company":"google","family":"Gemini","model":"gemini-1.0-pro","place":13,"score":55.7,"version":"Gemini 1.0"}],"stars":"","top_company":"qwen","top_model":"Qwen2-VL 72B Instruct","top_model_family":"Qwen","top_model_id":"qwen2-vl-72b-instruct","top_model_version":"Qwen2 VL","top_open_source":1,"top_percent":77.9,"topic":"Egocentric video QA","total_models":1613,"url":""},{"bench_models":4,"description":"Egocentric video temporal reasoning benchmark.","github":"","has_recent":0,"id":"egotempo","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EgoTempo","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":67,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":65.4,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":58.1,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":51.7,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":67,"topic":"Egocentric temporal reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Complex instruction-following benchmark.","github":"","has_recent":0,"id":"eifbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EIFBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":66.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":50.1,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":48.6,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":47,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":44.7,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":66.7,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":27,"description":"Embodied spatial understanding benchmark evaluating navigation and localization.","github":"","has_recent":0,"id":"embspatialbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EmbSpatialBench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":84.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":2,"score":84.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":3,"score":84.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":4,"score":84.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":83.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":83.1,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":7,"score":82.9,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":8,"score":81.5,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":9,"score":81.3,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":10,"score":80.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":11,"score":79.6,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":79.1,"version":"Gemini 2.5"},{"company":"robobrain","family":"RoboBrain","model":"robobrain-2.0","place":13,"score":78.6,"version":"RoboBrain"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":14,"score":78.5,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":15,"score":77.4,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":16,"score":76.4,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":17,"score":75.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":18,"score":72.1,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":19,"score":72.1,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":20,"score":71.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":21,"score":69.2,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":22,"score":69.2,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":23,"score":67.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":24,"score":67.5,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":25,"score":66.3,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":26,"score":61.2,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":27,"score":50.7,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":84.5,"topic":"Spatial understanding","total_models":1613,"url":""},{"bench_models":6,"description":"EMMA benchmark for multimodal reasoning.","github":"","has_recent":0,"id":"emma","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EMMA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":66.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":61.7,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":59.5,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":59.4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":53.5,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":49.5,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":66.5,"topic":"Multimodal reasoning","total_models":1613,"url":""},{"bench_models":8,"description":"Composite capability benchmark capturing broad model performance (Enamel score).","github":"","has_recent":0,"id":"enamel","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Enamel","paper":"","placements":[{"company":"essential","family":"Rnj","model":"rnj-1","place":1,"score":49,"version":"Rnj 1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":2,"score":38.3,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":3,"score":36.8,"version":"Qwen3"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":4,"score":29.5,"version":"Codestral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":5,"score":28.4,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":6,"score":17.5,"version":"OLMo 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":7,"score":11.5,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":8,"score":9.2,"version":"Llama 3.1"}],"stars":"","top_company":"essential","top_model":"Rnj-1","top_model_family":"Rnj","top_model_id":"rnj-1","top_model_version":"Rnj 1","top_open_source":1,"top_percent":49,"topic":"Composite capability","total_models":1613,"url":""},{"bench_models":4,"description":"English code editing benchmark for applying conditional modifications.","github":"","has_recent":0,"id":"enconda-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EnConda-Bench","paper":"","placements":[{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":1,"score":21.5,"version":"Youtu-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":2,"score":16.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":3,"score":10.8,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":4,"score":3.5,"version":"SmollM3"}],"stars":"","top_company":"tencent","top_model":"Youtu-LLM-2B","top_model_family":"Youtu-LLM","top_model_id":"youtu-llm-2b","top_model_version":"Youtu-LLM","top_open_source":1,"top_percent":21.5,"topic":"Code editing","total_models":1613,"url":""},{"bench_models":9,"description":"Encyclopedic knowledge evaluation benchmark.","github":"","has_recent":0,"id":"encyclo-k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Encyclo-K","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":1,"score":65.7,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":64.9,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":3,"score":64.5,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":63.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":5,"score":61,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":6,"score":60,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":58,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":8,"score":53,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":52.1,"version":"Seed 2.0"}],"stars":"","top_company":"bytedance","top_model":"Seed2.0 Pro","top_model_family":"Seed","top_model_id":"seed2.0-pro","top_model_version":"Seed 2.0","top_open_source":0,"top_percent":65.7,"topic":"Encyclopedic knowledge","total_models":1613,"url":""},{"bench_models":15,"description":"Challenging puzzle benchmark.","github":"","has_recent":0,"id":"enigmaval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EnigmaEval","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":17.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":15.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":14.5,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":12.4,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":5,"score":11.7,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":8.2,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4","place":7,"score":7.8,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":6,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":5.6,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":10,"score":5.2,"version":"Grok 4.1"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":11,"score":2.9,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":12,"score":2.7,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":13,"score":2.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":14,"score":0.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":15,"score":0.8,"version":"GPT-4o"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":17.8,"topic":"Challenging puzzles","total_models":1613,"url":""},{"bench_models":5,"description":"Enterprise retrieval-augmented generation evaluation covering internal knowledge bases.","github":"","has_recent":0,"id":"enterprise-rag","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Enterprise RAG","paper":"","placements":[{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":1,"score":69.2,"version":"Apriel Nemotron"},{"company":"openai","family":"o1","model":"o1-mini","place":2,"score":66.5,"version":"o1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":3,"score":65.2,"version":"QwQ"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":4,"score":52.1,"version":"EXAONE 4.0"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":5,"score":11.1,"version":"Llama 3.1"}],"stars":"","top_company":"servicenow","top_model":"Apriel Nemotron 15B Thinker","top_model_family":"Apriel","top_model_id":"apriel-nemotron-15b-thinker","top_model_version":"Apriel Nemotron","top_open_source":1,"top_percent":69.2,"topic":"Retrieval-augmented generation","total_models":1613,"url":""},{"bench_models":21,"description":"General reasoning benchmark assessing equation/logic capabilities.","github":"https://github.com/EQ-bench","has_recent":0,"id":"eqbench","leaderboard":"https://eqbench.com/eqbench-v2.html","lower_is_better":false,"metric":"percent","name":"EQ-Bench","paper":"https://arxiv.org/abs/2312.06281","placements":[{"company":"menlo","family":"Jan","model":"jan-v1-2509","place":1,"score":85,"version":"Jan v1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":2,"score":83.41,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":3,"score":82.67,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking-2507","place":4,"score":82.61,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":5,"score":82.58,"version":"Claude 3.5"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":6,"score":82.52,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":7,"score":82.19,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":8,"score":82.04,"version":"GPT-4"},{"company":"rys","family":"RYS","model":"rys-xlarge-base","place":9,"score":81.67,"version":"RYS"},{"company":"openai","family":"GPT","model":"gpt-4","place":10,"score":81.32,"version":"GPT-4"},{"company":"rys","family":"RYS","model":"rys-xlarge","place":11,"score":80.69,"version":"RYS"},{"company":"nousresearch","family":"Hermes","model":"hermes-4-70b","place":12,"score":80.5,"version":"Hermes 4"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":13,"score":80.32,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":14,"score":79.91,"version":"Llama 3.2"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":15,"score":79.51,"version":"Hermes 3"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":16,"score":77.91,"version":"Solar Pro"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":17,"score":77.22,"version":"Phi 3.5 MoE"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":18,"score":76.78,"version":"Phi 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":19,"score":65.8,"version":"Llama 3.1"},{"company":"menlo","family":"Jan","model":"jan-v1-edge","place":20,"score":57.2,"version":"Jan v1"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":21,"score":54.53,"version":"Qwen3"}],"stars":"352","top_company":"menlo","top_model":"Jan v1 2509","top_model_family":"Jan","top_model_id":"jan-v1-2509","top_model_version":"Jan v1","top_open_source":1,"top_percent":85,"topic":"Reasoning","total_models":1613,"url":"https://eqbench.com/eqbench-v2.html"},{"bench_models":10,"description":"A benchmark measuring emotional intelligence in challenging roleplays, judged by Sonnet 3.7.","github":"https://github.com/EQ-bench","has_recent":0,"id":"eqbench-3","leaderboard":"https://eqbench.com/index.html","lower_is_better":false,"metric":"ELO","name":"EQ-Bench 3","paper":"https://arxiv.org/abs/2312.06281","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":1,"score":1554.9,"version":"Kimi K2"},{"company":"openai","family":"o3","model":"o3","place":2,"score":1500,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":1454.8,"version":"Gemini 2.5"},{"company":"nousresearch","family":"Hermes","model":"hermes-4-405b","place":4,"score":1438.1,"version":"Hermes 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":5,"score":1349,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":1342,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":7,"score":1297.4,"version":"GLM 4.5"},{"company":"openai","family":"o4","model":"o4-mini","place":8,"score":1277.6,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":9,"score":1276.9,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":10,"score":1261.8,"version":"Qwen3"}],"stars":"21","top_company":"moonshotai","top_model":"Kimi K2 Instruct","top_model_family":"Kimi","top_model_id":"kimi-k2-instruct","top_model_version":"Kimi K2","top_open_source":1,"top_percent":1554.9,"topic":"Emotional intelligence (roleplay)","total_models":1613,"url":"https://eqbench.com/index.html"},{"bench_models":39,"description":"Spatial recognition and reasoning QA benchmark (ERQA).","github":"","has_recent":0,"id":"erqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ERQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-flash","place":1,"score":71,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":70.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":70.2,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":67.5,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":5,"score":65.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":64.8,"version":"Qwen3.5"},{"company":"openai","family":"o3","model":"o3","place":7,"score":64,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":8,"score":62,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":60.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":60.7,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":11,"score":60.5,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":60,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":13,"score":59.8,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":14,"score":59.8,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":15,"score":58.8,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":16,"score":56.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":17,"score":56,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":55.3,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":19,"score":54,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":20,"score":54,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":21,"score":53.7,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":22,"score":52.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":23,"score":52.5,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":24,"score":51,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":25,"score":50.3,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":26,"score":50.1,"version":"Grok 4"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":27,"score":50,"version":"GLM 4.5V"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":28,"score":49.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":29,"score":48.8,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":30,"score":47,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":31,"score":47,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":32,"score":46.8,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":33,"score":46.3,"version":"Grok 4.1"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":34,"score":45.8,"version":"GLM 4.1V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":35,"score":45.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":36,"score":45.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":37,"score":45,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":38,"score":44.6,"version":"Claude 4.5"},{"company":"stepfun","family":"Step","model":"step-3","place":39,"score":44.5,"version":"Step 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":40,"score":43,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":41,"score":41.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":42,"score":41.3,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":43,"score":40.5,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":44,"score":39.3,"version":"Claude 4"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":45,"score":37.5,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":46,"score":36.3,"version":"Claude 4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":47,"score":36,"version":"Kimi VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":48,"score":35.2,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":49,"score":28.3,"version":"Qwen3 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Flash","top_model_family":"Gemini","top_model_id":"gemini-3-flash","top_model_version":"Gemini 3","top_open_source":0,"top_percent":71,"topic":"Spatial reasoning","total_models":1613,"url":""},{"bench_models":34,"description":"Measures performance of LLM code evaluation, including runtime, memory, and efficiency metrics.","github":"https://github.com/evalplus/evalplus","has_recent":0,"id":"evalperf","leaderboard":"https://evalplus.github.io/evalperf.html","lower_is_better":false,"metric":"percent","name":"EvalPerf","paper":"https://openreview.net/forum?id=IBCBMeAhmC","placements":[{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":1,"score":100,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-chat-v2.5","place":2,"score":97,"version":"DeepSeek V2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20240620","place":3,"score":93.9,"version":"Claude 3.5"},{"company":"openai","family":"o1","model":"o1-preview-2024-09-12","place":4,"score":90.9,"version":"o1"},{"company":"mistral","family":"Mistral","model":"mistral-large-instruct-2407","place":5,"score":87.9,"version":"Mistral"},{"company":"openai","family":"o1","model":"o1-mini-2024-09-12-high","place":6,"score":84.8,"version":"o1"},{"company":"openai","family":"o1","model":"o1-mini-2024-09-12","place":7,"score":81.8,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":8,"score":78.8,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-chat-v3","place":9,"score":75.8,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":10,"score":72.7,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":11,"score":69.7,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":12,"score":66.7,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":13,"score":60.6,"version":"Qwen2.5"},{"company":"01ai","family":"Yi 1.5","model":"yi-coder-9b-chat","place":14,"score":57.6,"version":"Yi 1.5"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":15,"score":57.6,"version":"Magicoder-S-DS-6.7B"},{"company":"qwen","family":"QwQ","model":"qwq-32b-preview","place":16,"score":54.5,"version":"QwQ"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":17,"score":54.5,"version":"DeepSeek Coder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":18,"score":51.5,"version":"DeepSeek Coder V2"},{"company":"mistral","family":"Mistral","model":"mistral-small-instruct-2409","place":19,"score":42.4,"version":"Mistral"},{"company":"mistral","family":"Codestral","model":"codestral-22b-v0.1","place":20,"score":39.4,"version":"Codestral v0.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":21,"score":39.4,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen2.5-14b-instruct","place":22,"score":36.4,"version":"Qwen2.5"},{"company":"xai","family":"Grok","model":"grok-beta","place":23,"score":33.3,"version":"Grok"},{"company":"map","family":"OpenCoder","model":"opencoder-8b-instruct","place":24,"score":30.3,"version":"OpenCoder"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":25,"score":27.3,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":26,"score":24.2,"version":"Gemma 2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":27,"score":24.2,"version":"Gemini 1.5"},{"company":"mistral","family":"Mistral","model":"mistral-nemo-instruct-2407","place":28,"score":21.2,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":29,"score":15.2,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b-chat","place":30,"score":12.1,"version":"CodeQwen1.5"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":31,"score":9.1,"version":"Phi 3"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-70b-instruct-hf","place":32,"score":6.1,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":33,"score":3,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"meta-llama-3-8b-instruct","place":34,"score":0,"version":"Llama 3"}],"stars":"","top_company":"openai","top_model":"GPT-4o (2024-08-06)","top_model_family":"GPT","top_model_id":"gpt-4o-2024-08-06","top_model_version":"GPT-4o","top_open_source":0,"top_percent":100,"topic":"Code evaluation performance","total_models":1613,"url":"https://evalplus.github.io/evalperf.html"},{"bench_models":27,"description":"Aggregated code evaluation suite from EvalPlus.","github":"","has_recent":0,"id":"evalplus","leaderboard":"https://evalplus.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"EvalPlus","paper":"","placements":[{"company":"openai","family":"o1","model":"o1-mini","place":1,"score":89,"version":"o1"},{"company":"openai","family":"o1","model":"o1-preview","place":2,"score":89,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":3,"score":87.2,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":4,"score":87.2,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":5,"score":86.6,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":6,"score":86.6,"version":"GPT-4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":7,"score":83.5,"version":"DeepSeek V2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":8,"score":83.5,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":9,"score":82.3,"version":"DeepSeek Coder V2"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":10,"score":81.7,"version":"Claude 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":11,"score":80.3,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":12,"score":78.1,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":13,"score":77.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":14,"score":72.23,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":15,"score":72.22,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":16,"score":72.05,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":17,"score":71.45,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":18,"score":68.9,"version":"Falcon H1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":19,"score":65.6,"version":"DeepSeek V3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":20,"score":65.5,"version":"Llama 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":21,"score":64.9,"version":"Kimi Linear"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-instruct","place":22,"score":55.4,"version":"CoDA"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":23,"score":54.9,"version":"Dream 7B"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":24,"score":53.7,"version":"Dream"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":25,"score":50.3,"version":"Moonlight"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-base","place":26,"score":34.9,"version":"CoDA"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":27,"score":30.2,"version":"LLaDA"}],"stars":"1577","top_company":"openai","top_model":"o1 Mini","top_model_family":"o1","top_model_id":"o1-mini","top_model_version":"o1","top_open_source":0,"top_percent":89,"topic":"Code generation","total_models":1613,"url":"https://github.com/evalplus/evalplus"},{"bench_models":2,"description":"EVG document OCR benchmark evaluating recognition accuracy and layout extraction.","github":"","has_recent":0,"id":"evg","leaderboard":"","lower_is_better":true,"metric":"percent","name":"EVG","paper":"","placements":[{"company":"bytedance","family":"Dolphin","model":"dolphin-1.5","place":1,"score":2.98,"version":"Dolphin"},{"company":"bytedance","family":"Dolphin","model":"dolphin","place":2,"score":4.24,"version":"Dolphin"}],"stars":"","top_company":"bytedance","top_model":"Dolphin 1.5","top_model_family":"Dolphin","top_model_id":"dolphin-1.5","top_model_version":"Dolphin","top_open_source":1,"top_percent":2.98,"topic":"Document OCR","total_models":1613,"url":""},{"bench_models":5,"description":"Multilingual character-level evaluation benchmark.","github":"","has_recent":0,"id":"execute","leaderboard":"","lower_is_better":false,"metric":"percent","name":"EXECUTE","paper":"","placements":[{"company":"allenai","family":"Bolmo","model":"bolmo-7b","place":1,"score":71.6,"version":"Bolmo 7B"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b","place":2,"score":55.1,"version":"OLMo 3"},{"company":"meta","family":"BLT","model":"blt-7b","place":3,"score":46.3,"version":"BLT 7B"},{"company":"hku","family":"EvaByte","model":"evabyte-6.5b","place":4,"score":43.8,"version":"EvaByte 6.5B"},{"company":"alephalpha","family":"TFree","model":"tfree-hat-pretrained-7b-base","place":5,"score":41.6,"version":"TFree Hat"}],"stars":"","top_company":"allenai","top_model":"Bolmo 7B","top_model_family":"Bolmo","top_model_id":"bolmo-7b","top_model_version":"Bolmo 7B","top_open_source":1,"top_percent":71.6,"topic":"Multilingual character tasks","total_models":1613,"url":""},{"bench_models":9,"description":"Comprehensive factuality benchmark suite covering held-out internal grounding, parametric knowledge, multimodal understanding, and search retrieval benchmarks.","github":"","has_recent":0,"id":"facts-benchmark-suite","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FACTS Benchmark Suite","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":70.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":63.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":61.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":4,"score":61.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":5,"score":50.8,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":6,"score":50.4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":7,"score":50.4,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":8,"score":48.9,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":9,"score":42.1,"version":"Grok 4.1"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":70.5,"topic":"Held out internal grounding, parametric, MM, and search retrieval benchmarks","total_models":1613,"url":""},{"bench_models":24,"description":"Grounded factuality benchmark evaluating model alignment with source facts.","github":"","has_recent":0,"id":"facts-grounding","leaderboard":"https://www.kaggle.com/benchmarks/google/facts-grounding","lower_is_better":false,"metric":"percent","name":"FACTS Grounding","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":1,"score":88.5,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":87.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":3,"score":86.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":4,"score":84.6,"version":"Gemini 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":5,"score":83.6,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":6,"score":82.9,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":7,"score":82.8,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":8,"score":80,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":9,"score":79.2,"version":"GPT-4.1"},{"company":"ai21","family":"Jamba","model":"jamba-mini-1.7","place":10,"score":79,"version":"Jamba Mini"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":11,"score":75.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":12,"score":75.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":13,"score":74.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":14,"score":74.9,"version":"Gemma 3"},{"company":"ai21","family":"Jamba","model":"jamba-mini-1.6","place":15,"score":72.7,"version":"Jamba Mini"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":16,"score":70.1,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":17,"score":70.1,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":18,"score":68.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":19,"score":68.3,"version":"DeepSeek V3"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":20,"score":62.4,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":21,"score":62,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":22,"score":43.8,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":23,"score":36.4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":24,"score":36.4,"version":"Gemma 3"}],"stars":"","top_company":"moonshotai","top_model":"Kimi K2 Instruct","top_model_family":"Kimi","top_model_id":"kimi-k2-instruct","top_model_version":"Kimi K2","top_open_source":1,"top_percent":88.5,"topic":"Grounding / factuality","total_models":1613,"url":"https://goo.gle/FACTS_paper"},{"bench_models":11,"description":"Measures hallucination rate on an open-source prompt suite; lower is better.","github":"","has_recent":0,"id":"factscore","leaderboard":"","lower_is_better":true,"metric":"percent","name":"FActScore","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":1,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":5.7,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":3,"score":50.4,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":4,"score":62.4,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":5,"score":71.2,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":90.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":91.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":8,"score":91.9,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":9,"score":92,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":92.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":11,"score":96.1,"version":"GPT-5"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":1,"topic":"Hallucination rate on open-source prompts","total_models":1613,"url":""},{"bench_models":8,"description":"FaithJudge hallucination rate with 1-hallucination metric (lower is better).","github":"","has_recent":0,"id":"faithjudge-1-hallu","leaderboard":"","lower_is_better":true,"metric":"percent","name":"FaithJudge (1-Hallu.)","paper":"","placements":[{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":1,"score":56,"version":"Moonlight"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":2,"score":64.2,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":3,"score":75.7,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":4,"score":83,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":5,"score":83.4,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":6,"score":91,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":7,"score":92.6,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":8,"score":93.2,"version":"Gemini 2.5"}],"stars":"","top_company":"moonshotai","top_model":"Moonlight-Instruct","top_model_family":"Moonlight","top_model_id":"moonlight-instruct","top_model_version":"Moonlight","top_open_source":1,"top_percent":56,"topic":"Hallucination detection","total_models":1613,"url":""},{"bench_models":76,"description":"","github":"","has_recent":0,"id":"faix-agent","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score Agent","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":100,"version":"Claude 4.5"},{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":2,"score":90.67,"version":"Holo1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":60,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":4,"score":58.8,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":5,"score":58.4,"version":"Grok 4"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":6,"score":56,"version":"Holo1.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":7,"score":54.44,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":8,"score":52,"version":"Claude 3.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":9,"score":49.33,"version":"UI-Venus"},{"company":"openai","family":"o3","model":"o3-low","place":10,"score":47.67,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":11,"score":46.67,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":12,"score":42.91,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":13,"score":42,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":14,"score":41.68,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":15,"score":41.14,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":16,"score":40,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":17,"score":36,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":18,"score":31.43,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":19,"score":29.6,"version":"Claude 3.7"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":20,"score":24.8,"version":"MiniMax M2"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":21,"score":24,"version":"Holo1.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":22,"score":24,"version":"Seed 1.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":23,"score":21.33,"version":"UI-Venus"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":24,"score":20,"version":"Grok Code"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":25,"score":19.2,"version":"Qwen2.5 VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":26,"score":18.67,"version":"UI-TARS"},{"company":"openai","family":"o4","model":"o4-mini","place":27,"score":18.5,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":28,"score":16,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":29,"score":16,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":30,"score":16,"version":"GPT-4o"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":31,"score":16,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":32,"score":16,"version":"Qwen3 VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":33,"score":15,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":34,"score":14.4,"version":"GLM 4.6"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":35,"score":13.6,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":36,"score":13.33,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":37,"score":12,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":38,"score":12,"version":"MiniMax M1"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":39,"score":11.56,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":40,"score":9.33,"version":"Qwen3 VL"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":41,"score":9,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":42,"score":8,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":43,"score":8,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":44,"score":8,"version":"Gemini 2.5"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":45,"score":8,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":46,"score":5.33,"version":"Qwen3 Next"},{"company":"zai","family":"GLM","model":"glm-4.5","place":47,"score":5.14,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":48,"score":4.92,"version":"Kimi K2"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":49,"score":4,"version":"Apriel 1.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":50,"score":4,"version":"MiniMax M1"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":51,"score":4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":52,"score":2.86,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":53,"score":2.67,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":54,"score":2,"version":"DeepSeek V3.1"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":55,"score":1.6,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":56,"score":1.33,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":57,"score":0.8,"version":"GPT-OSS"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":58,"score":0,"version":"EXAONE 4.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":59,"score":0,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":60,"score":0,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":61,"score":0,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":62,"score":0,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":63,"score":0,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":64,"score":0,"version":"Llama 4"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":65,"score":0,"version":"Llama Nemotron v1"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":66,"score":0,"version":"Magistral 1.2"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":67,"score":0,"version":"Magistral 1.2"},{"company":"qwen","family":"Qwen","model":"qwen-3-30b-a3b","place":68,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":69,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-coder-480b-a35b-instruct","place":70,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":71,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":72,"score":0,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":73,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":74,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":75,"score":0,"version":"Qwen3 Next"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":76,"score":0,"version":"Seed 1.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":100,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":163,"description":"","github":"","has_recent":0,"id":"faix-code","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score Code","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":100,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":2,"score":63.27,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":57,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":53.33,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":5,"score":49.14,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":6,"score":43.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":7,"score":42,"version":"Claude 4"},{"company":"openai","family":"o1","model":"o1","place":8,"score":40.33,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":9,"score":40,"version":"GPT-5.1"},{"company":"openai","family":"o3","model":"o3-low","place":10,"score":37.6,"version":"o3"},{"company":"openai","family":"o1","model":"o1-mini","place":11,"score":35.56,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":12,"score":35.2,"version":"Claude 4.1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":13,"score":31,"version":"ERNIE 4.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":14,"score":30.67,"version":"OLMo 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":15,"score":30,"version":"Motif 2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":29.82,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":17,"score":29.6,"version":"Grok 4"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":18,"score":29.33,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":19,"score":29.33,"version":"Qwen3 VL"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":20,"score":26.67,"version":"MiniMax M2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":21,"score":26.13,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":22,"score":24,"version":"Qwen3 Next"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":23,"score":23.33,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":24,"score":22.4,"version":"DeepSeek R1"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":25,"score":22,"version":"Grok 4"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":26,"score":20.8,"version":"Llama 3.1"},{"company":"mbzuai","family":"K2","model":"k2-think","place":27,"score":20,"version":"K2-THINK"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":28,"score":19,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-4","place":29,"score":18.67,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":30,"score":18.2,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":31,"score":17.82,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":32,"score":17.2,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":33,"score":17.09,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":34,"score":16.57,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-coder-480b-a35b-instruct","place":35,"score":16.36,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":36,"score":16,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":37,"score":16,"version":"Qwen3 VL"},{"company":"openai","family":"o3","model":"o3-mini-low","place":38,"score":15,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":39,"score":14.8,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":40,"score":14.5,"version":"DeepSeek V2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":41,"score":14.4,"version":"SmollM3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":42,"score":14.29,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":43,"score":13.6,"version":"DeepSeek V3.1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":44,"score":13.33,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":45,"score":13.33,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":46,"score":13.07,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":47,"score":13,"version":"GLM 4.5"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":48,"score":12.57,"version":"Llama 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":49,"score":12.44,"version":"DeepSeek V3.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":50,"score":10.67,"version":"MiniMax M1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":51,"score":10.17,"version":"DeepSeek V3"},{"company":"openai","family":"o4","model":"o4-mini","place":52,"score":9.33,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":53,"score":9.2,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":54,"score":9.14,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":55,"score":9.09,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":56,"score":8,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":57,"score":8,"version":"Kimi K2"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":58,"score":8,"version":"MiniMax M1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":59,"score":8,"version":"QwQ"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":60,"score":7.5,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen-3-30b-a3b","place":61,"score":7.33,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":62,"score":7,"version":"GLM 4.6"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":63,"score":6.67,"version":"Mistral 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":64,"score":6.5,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":65,"score":6.4,"version":"Claude 3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":66,"score":6,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":67,"score":6,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":68,"score":4.57,"version":"Claude 3.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":69,"score":4,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":70,"score":4,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":71,"score":3.43,"version":"Qwen2.5"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":72,"score":2.67,"version":"Codestral"},{"company":"google","family":"Gemma","model":"gemma-3","place":73,"score":2.29,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":74,"score":2,"version":"Gemma 2"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":75,"score":2,"version":"Granite 3.3"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":76,"score":2,"version":"Grok Code"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":77,"score":2,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":78,"score":2,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":79,"score":2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":80,"score":2,"version":"Qwen3 VL"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":81,"score":1.78,"version":"Grok 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":82,"score":1.78,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":83,"score":1.33,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":84,"score":1,"version":"OLMo 2"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":85,"score":0.8,"version":"DeepSeek Coder"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":86,"score":0.67,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":87,"score":0.44,"version":"Qwen2.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":88,"score":0,"version":"Apriel 1.5"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":89,"score":0,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":90,"score":0,"version":"Claude 4.5"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":91,"score":0,"version":"CodeLlama-7B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":92,"score":0,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":93,"score":0,"version":"DeepSeek R1"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":94,"score":0,"version":"Dream"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":95,"score":0,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":96,"score":0,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":97,"score":0,"version":"EXAONE Deep"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":98,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":99,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":100,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":101,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":102,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":103,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":104,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":105,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":106,"score":0,"version":"Falcon 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":107,"score":0,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":108,"score":0,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":109,"score":0,"version":"Gemini 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":110,"score":0,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":111,"score":0,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":112,"score":0,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo","place":113,"score":0,"version":"GPT-3.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":114,"score":0,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":115,"score":0,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":116,"score":0,"version":"GPT-4.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":117,"score":0,"version":"GPT-5"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":118,"score":0,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":119,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":120,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":121,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":122,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":123,"score":0,"version":"Granite 4.0"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":124,"score":0,"version":"Grok 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":125,"score":0,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":126,"score":0,"version":"Kimi Linear"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":127,"score":0,"version":"LFM2"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":128,"score":0,"version":"LFM2"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":129,"score":0,"version":"LLaDA"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":130,"score":0,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":131,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":132,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":133,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":134,"score":0,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":135,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":136,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":137,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":138,"score":0,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":139,"score":0,"version":"Llama 4"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":140,"score":0,"version":"Llama Nemotron v1"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":141,"score":0,"version":"Magistral 1.2"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":142,"score":0,"version":"Magistral 1.2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":143,"score":0,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":144,"score":0,"version":"Mistral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":145,"score":0,"version":"Mixtral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":146,"score":0,"version":"Mixtral"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":147,"score":0,"version":"MobileLLM"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":148,"score":0,"version":"Moonlight"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":149,"score":0,"version":"Motif 2"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":150,"score":0,"version":"Ouro"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":151,"score":0,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-3b-instruct","place":152,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":153,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":154,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":155,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":156,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":157,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":158,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":159,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":160,"score":0,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b","place":161,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":162,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":163,"score":0,"version":"Qwen3 VL"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":100,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":175,"description":"","github":"","has_recent":0,"id":"faix-math","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score Math","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":1,"score":100,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":63.33,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":3,"score":61.33,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":4,"score":54,"version":"Gemini 2.0"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":5,"score":50,"version":"Claude 4.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":6,"score":50,"version":"Ling 1T"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":7,"score":48,"version":"GLM 4.5V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":44.33,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":9,"score":44,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":10,"score":40,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":11,"score":38.67,"version":"Qwen3 VL"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":12,"score":37,"version":"OLMo 3"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":13,"score":36,"version":"dots.vlm1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":14,"score":33.33,"version":"DeepSeek V3.1"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":15,"score":33.33,"version":"Grok 4"},{"company":"openai","family":"o4","model":"o4-mini","place":16,"score":33.33,"version":"o4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":17,"score":33,"version":"Kimi K2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":18,"score":32,"version":"ERNIE 4.5 VL"},{"company":"openai","family":"o3","model":"o3-low","place":19,"score":31.43,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":20,"score":30.4,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":21,"score":30,"version":"GPT-4.5"},{"company":"stepfun","family":"Step","model":"step-3","place":22,"score":28,"version":"Step 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":23,"score":26,"version":"GLM 4.1V"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":24,"score":24.67,"version":"GPT-OSS"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":25,"score":22,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":26,"score":22,"version":"Qwen3 Next"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":27,"score":21.33,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":28,"score":20.67,"version":"Claude 4"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":29,"score":20,"version":"Grok 3"},{"company":"openai","family":"o3","model":"o3-mini-low","place":30,"score":20,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":31,"score":19,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":32,"score":18.67,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":33,"score":18,"version":"Apriel Nemotron"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":34,"score":18,"version":"EXAONE 4.0"},{"company":"openai","family":"o1","model":"o1","place":35,"score":18,"version":"o1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":36,"score":18,"version":"OLMo 3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":37,"score":17.33,"version":"QwQ"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":38,"score":16.8,"version":"Motif 2"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":39,"score":16.67,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":40,"score":16,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3","place":41,"score":15,"version":"Gemma 3"},{"company":"mbzuai","family":"K2","model":"k2-think","place":42,"score":15,"version":"K2-THINK"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":43,"score":14.4,"version":"GPT-OSS"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":44,"score":13.33,"version":"InternVL 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":45,"score":12,"version":"DeepSeek R1"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":46,"score":12,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-3b-instruct","place":47,"score":12,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":48,"score":11.56,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":49,"score":11.2,"version":"Claude 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking","place":50,"score":11.2,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":51,"score":11.11,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":52,"score":10,"version":"DeepSeek V3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":53,"score":10,"version":"ERNIE 4.5"},{"company":"openai","family":"o1","model":"o1-mini","place":54,"score":10,"version":"o1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":55,"score":10,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":56,"score":9.2,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":57,"score":9,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":58,"score":8,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":59,"score":8,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":60,"score":8,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":61,"score":8,"version":"Gemini 2.5"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":62,"score":8,"version":"Gemma 3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":63,"score":8,"version":"LFM2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":64,"score":7.33,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen-3-30b-a3b","place":65,"score":7,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":66,"score":6.86,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":67,"score":6.29,"version":"GPT-4.1"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":68,"score":6,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":69,"score":6,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":70,"score":5.33,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":71,"score":5.33,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":72,"score":4.44,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":73,"score":4,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":74,"score":4,"version":"Gemma 3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":75,"score":4,"version":"Magistral 1.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":76,"score":4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":77,"score":4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":78,"score":3.43,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":79,"score":3.43,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":80,"score":3.2,"version":"Falcon H1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":81,"score":2.67,"version":"Claude 3.7"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":82,"score":2.67,"version":"Llama 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":83,"score":2.29,"version":"Gemini 2.5"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":84,"score":2,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":85,"score":2,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":86,"score":2,"version":"Qwen2.5 VL"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":87,"score":1.6,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":88,"score":1.33,"version":"Gemma 2"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":89,"score":1.33,"version":"MiniMax M1"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":90,"score":1,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":91,"score":1,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":92,"score":0.8,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":93,"score":0.57,"version":"Llama 3.2"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":94,"score":0,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":95,"score":0,"version":"Claude 3.5"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":96,"score":0,"version":"CodeLlama-7B"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":97,"score":0,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":98,"score":0,"version":"EXAONE Deep"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":99,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":100,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":101,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":102,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":103,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":104,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":105,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":106,"score":0,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":107,"score":0,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":108,"score":0,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":109,"score":0,"version":"Gemma 3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":110,"score":0,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":111,"score":0,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":112,"score":0,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":113,"score":0,"version":"GPT-5"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":114,"score":0,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":115,"score":0,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":116,"score":0,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":117,"score":0,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":118,"score":0,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":119,"score":0,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":120,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":121,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":122,"score":0,"version":"Granite 4.0"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":123,"score":0,"version":"Grok 3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":124,"score":0,"version":"InternVL 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":125,"score":0,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":126,"score":0,"version":"Kimi Linear"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":127,"score":0,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":128,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":129,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":130,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":131,"score":0,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":132,"score":0,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":133,"score":0,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":134,"score":0,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":135,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":136,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":137,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":138,"score":0,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":139,"score":0,"version":"Llama 4"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":140,"score":0,"version":"Llama Nemotron v1"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":141,"score":0,"version":"LLaVA OneVision"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":142,"score":0,"version":"Magistral 1.2"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":143,"score":0,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":144,"score":0,"version":"Marin 32B"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":145,"score":0,"version":"MiniMax M1"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":146,"score":0,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":147,"score":0,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":148,"score":0,"version":"Mistral 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":149,"score":0,"version":"Mistral 3.1"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":150,"score":0,"version":"Mistral 3.2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":151,"score":0,"version":"Mixtral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":152,"score":0,"version":"Mixtral"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":153,"score":0,"version":"Moonlight"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":154,"score":0,"version":"Motif 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":155,"score":0,"version":"Nemotron Nano v2"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":156,"score":0,"version":"Ouro"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":157,"score":0,"version":"Ouro"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":158,"score":0,"version":"Pixtral"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":159,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":160,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":161,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":162,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":163,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":164,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":165,"score":0,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":166,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":167,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b","place":168,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":169,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":170,"score":0,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":171,"score":0,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":172,"score":0,"version":"Seed 1.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":173,"score":0,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":174,"score":0,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":175,"score":0,"version":"SmolLM2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-235B-A22B Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-235b-a22b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":100,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":82,"description":"","github":"","has_recent":0,"id":"faix-ocr","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score OCR","paper":"","placements":[{"company":"openai","family":"o3","model":"o3-low","place":1,"score":80,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":2,"score":73.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":3,"score":44.36,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":4,"score":37,"version":"Qwen2 VL"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":5,"score":34.67,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":33.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":7,"score":29.33,"version":"GPT-5"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":8,"score":28,"version":"Molmo"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl","place":9,"score":27.43,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":10,"score":26.4,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":11,"score":26.29,"version":"dots.vlm1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":12,"score":25,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":13,"score":24.73,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":14,"score":21.71,"version":"Qwen2.5 VL"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":15,"score":20,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":16,"score":20,"version":"Llama 4"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":17,"score":19,"version":"InternVL 2"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":18,"score":15.33,"version":"Manzano"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":19,"score":14,"version":"ERNIE 4.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":20,"score":12.8,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":21,"score":12.57,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":22,"score":12,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":23,"score":10.5,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":24,"score":10,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b-instruct","place":25,"score":9.6,"version":"Qwen2 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":26,"score":6.86,"version":"InternVL"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":27,"score":6,"version":"Llama 3.2"},{"company":"stepfun","family":"Step","model":"step-3","place":28,"score":6,"version":"Step 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":29,"score":5.33,"version":"Gemini 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":30,"score":5.33,"version":"InternVL 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":31,"score":4.36,"version":"Qwen3 VL"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":32,"score":4,"version":"Llama 4"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":33,"score":4,"version":"Molmo"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":34,"score":4,"version":"Pixtral"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":35,"score":3.64,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":36,"score":3.27,"version":"Claude 4"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":37,"score":2,"version":"LLaVA OneVision"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking","place":38,"score":1.6,"version":"Kimi VL"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":39,"score":1.33,"version":"Mistral 3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":40,"score":0.73,"version":"Gemini 2.5"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":41,"score":0.67,"version":"Manzano"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":42,"score":0,"version":"Bagel"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":43,"score":0,"version":"BLIP-3O"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":44,"score":0,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":45,"score":0,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":46,"score":0,"version":"Cambrian"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":47,"score":0,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":48,"score":0,"version":"Claude 3"},{"company":"baai","family":"Emu","model":"emu3-8b","place":49,"score":0,"version":"Emu3"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":50,"score":0,"version":"FastVLM"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":51,"score":0,"version":"FastVLM"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":52,"score":0,"version":"FastVLM"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":53,"score":0,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":54,"score":0,"version":"Gemini 2.5"},{"company":"google","family":"Gemma","model":"gemma-3","place":55,"score":0,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":56,"score":0,"version":"Gemma 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":57,"score":0,"version":"GLM 4.1V"},{"company":"openai","family":"GPT","model":"gpt-4v","place":58,"score":0,"version":"GPT-4V"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":59,"score":0,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":60,"score":0,"version":"InternVL 2"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":61,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":62,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":63,"score":0,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":64,"score":0,"version":"InternVL 3.5"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":65,"score":0,"version":"Janus Pro"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":66,"score":0,"version":"Llama 3.2"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":67,"score":0,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":68,"score":0,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":69,"score":0,"version":"LLaVA OneVision"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":70,"score":0,"version":"MiniCPM V 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":71,"score":0,"version":"Mistral 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":72,"score":0,"version":"Mistral 3.2"},{"company":"apple","family":"MM1","model":"mm1-30b","place":73,"score":0,"version":"MM1"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":74,"score":0,"version":"MM1.5"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":75,"score":0,"version":"MM1.5"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":76,"score":0,"version":"MolmoE"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":77,"score":0,"version":"PaliGemma"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":78,"score":0,"version":"Phi-3"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":79,"score":0,"version":"Phi 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":80,"score":0,"version":"Qwen2.5 Omni"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":81,"score":0,"version":"Qwen2.5 VL"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":82,"score":0,"version":"xGen-MM"}],"stars":"","top_company":"openai","top_model":"o3 (Low)","top_model_family":"o3","top_model_id":"o3-low","top_model_version":"o3","top_open_source":0,"top_percent":80,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":53,"description":"","github":"","has_recent":0,"id":"faix-safety","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score Safety","paper":"","placements":[{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":1,"score":70,"version":"Granite 3.3"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":2,"score":68,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":3,"score":66,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":4,"score":60,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5","place":5,"score":59,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-low","place":6,"score":56,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":7,"score":50,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":8,"score":48,"version":"Llama 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":9,"score":45.33,"version":"Claude 4.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":10,"score":36,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":11,"score":36,"version":"Granite 3.3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":12,"score":36,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":13,"score":33,"version":"Claude 4"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":14,"score":33,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":15,"score":30,"version":"Gemini 2.0"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":16,"score":28,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":17,"score":26.4,"version":"Llama 3.3"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":18,"score":24,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":19,"score":24,"version":"GPT-4o"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":20,"score":24,"version":"Granite 4.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":21,"score":24,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":22,"score":22.86,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":23,"score":22.67,"version":"Llama 3.1"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":24,"score":22,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":25,"score":20.8,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":26,"score":20,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":27,"score":20,"version":"Gemini 2.5"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":28,"score":20,"version":"Mixtral"},{"company":"openai","family":"o3","model":"o3-mini-low","place":29,"score":20,"version":"o3"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":30,"score":18,"version":"Codestral"},{"company":"openai","family":"o4","model":"o4-mini","place":31,"score":16,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":32,"score":13.33,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":33,"score":13,"version":"Claude 3.5"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":34,"score":12,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":35,"score":12,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":36,"score":12,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":37,"score":6,"version":"GPT-4.5"},{"company":"openai","family":"o1","model":"o1","place":38,"score":4,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":39,"score":2.67,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":40,"score":1.6,"version":"Llama 3.1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":41,"score":0,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":42,"score":0,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":43,"score":0,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":44,"score":0,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":45,"score":0,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":46,"score":0,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":47,"score":0,"version":"GPT-4.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":48,"score":0,"version":"Granite 3.2"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":49,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":50,"score":0,"version":"Llama 3.2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":51,"score":0,"version":"Mistral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":52,"score":0,"version":"Mixtral"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":53,"score":0,"version":"Qwen3"}],"stars":"","top_company":"IBM","top_model":"Granite 3.3 8B Instruct","top_model_family":"Granite","top_model_id":"granite-3.3-8b-instruct","top_model_version":"Granite 3.3","top_open_source":1,"top_percent":70,"topic":"Composite safety index","total_models":1613,"url":""},{"bench_models":226,"description":"","github":"","has_recent":0,"id":"faix-stem","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score STEM","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":1,"score":100,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":66.13,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":3,"score":54,"version":"Gemini 2.0"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":4,"score":50,"version":"FastVLM"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":5,"score":50,"version":"Ling 1T"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":6,"score":46,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":7,"score":44,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":8,"score":44,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":42.59,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":10,"score":40,"version":"GLM 4.5V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":11,"score":38.86,"version":"Claude 4.1"},{"company":"openai","family":"o3","model":"o3-low","place":12,"score":38,"version":"o3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":13,"score":37,"version":"Llama TFree HAT"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":14,"score":36,"version":"ERNIE 4.5 VL"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":15,"score":36,"version":"Molmo"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":16,"score":33.33,"version":"DeepSeek V3.1"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":17,"score":33.33,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":18,"score":32,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":19,"score":32,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":20,"score":31.43,"version":"Claude 4"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":21,"score":30,"version":"FastVLM"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":22,"score":29.6,"version":"Llama 3.1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":23,"score":29.6,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":24,"score":29,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":25,"score":27,"version":"dots.vlm1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":26,"score":24,"version":"Claude 4.5"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":27,"score":24,"version":"Manzano"},{"company":"openai","family":"o4","model":"o4-mini","place":28,"score":23.2,"version":"o4"},{"company":"stepfun","family":"Step","model":"step-3","place":29,"score":22.4,"version":"Step 3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":30,"score":21.14,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":31,"score":20.8,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":32,"score":20.5,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":33,"score":20,"version":"GPT-4.5"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":34,"score":18,"version":"Apriel Nemotron"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":35,"score":18,"version":"EXAONE 4.0"},{"company":"openai","family":"o1","model":"o1","place":36,"score":18,"version":"o1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":37,"score":17.6,"version":"OLMo 3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":38,"score":17.33,"version":"QwQ"},{"company":"openai","family":"o3","model":"o3-mini-low","place":39,"score":16.67,"version":"o3"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":40,"score":16.4,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":41,"score":16.33,"version":"Claude 4"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":42,"score":16,"version":"Manzano"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":43,"score":15.2,"version":"Gemini 2.0"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":44,"score":15,"version":"Grok 3"},{"company":"mbzuai","family":"K2","model":"k2-think","place":45,"score":15,"version":"K2-THINK"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":46,"score":14.4,"version":"Llama 3.3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":47,"score":14.4,"version":"OLMo 3"},{"company":"google","family":"Gemma","model":"gemma-3","place":48,"score":14.15,"version":"Gemma 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":49,"score":14,"version":"Motif 2"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":50,"score":13.71,"version":"Claude 3.5"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":51,"score":13.33,"version":"MM1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":52,"score":13.33,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":53,"score":12,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":54,"score":12,"version":"Llama 3.1"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":55,"score":12,"version":"Llama 3.1"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":56,"score":10.67,"version":"Phi-3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":57,"score":10.67,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":58,"score":10,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":59,"score":10,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":60,"score":10,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":61,"score":9.6,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":62,"score":9.6,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking","place":63,"score":9.33,"version":"Kimi VL"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":64,"score":9.33,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":65,"score":9.33,"version":"Marin 32B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":66,"score":9,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":67,"score":9,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-3b-instruct","place":68,"score":9,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":69,"score":8.57,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":70,"score":8.36,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":71,"score":8,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":72,"score":8,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":73,"score":8,"version":"DeepSeek V3.2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":74,"score":8,"version":"ERNIE 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":75,"score":8,"version":"InternVL 3.5"},{"company":"openai","family":"o1","model":"o1-mini","place":76,"score":8,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":77,"score":7.5,"version":"GPT-4.1"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":78,"score":7.11,"version":"OLMo 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":79,"score":6.86,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":80,"score":6.55,"version":"Gemma 3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":81,"score":6.4,"version":"LFM2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":82,"score":6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":83,"score":6,"version":"Gemini 2.5"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":84,"score":5.71,"version":"Gemma 2"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":85,"score":5.33,"version":"Pixtral"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":86,"score":5.14,"version":"Qwen3 VL"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":87,"score":4.8,"version":"LFM2"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":88,"score":4.57,"version":"Claude 3.7"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":89,"score":4,"version":"Gemini 1.5"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":90,"score":4,"version":"Magistral 1.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":91,"score":4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":92,"score":3.64,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-30b-a3b","place":93,"score":3.56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":94,"score":3.43,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":95,"score":3.2,"version":"Gemma 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":96,"score":3.2,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":97,"score":3,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":98,"score":2.67,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":99,"score":2.67,"version":"Llama 4"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":100,"score":2.67,"version":"Nemotron Nano v2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":101,"score":2.67,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":102,"score":2.5,"version":"DeepSeek V3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":103,"score":2.29,"version":"Falcon H1"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":104,"score":2,"version":"Mixtral"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":105,"score":1.6,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":106,"score":1.45,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":107,"score":1.45,"version":"Llama 4"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":108,"score":1.33,"version":"MiniMax M1"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":109,"score":1.33,"version":"MM1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":110,"score":1.33,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":111,"score":1,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":112,"score":0.8,"version":"Gemma 2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":113,"score":0.8,"version":"Mixtral"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":114,"score":0.67,"version":"Mistral 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":115,"score":0.67,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":116,"score":0.44,"version":"Llama 3.2"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":117,"score":0,"version":"Bagel"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":118,"score":0,"version":"BLIP-3O"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":119,"score":0,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":120,"score":0,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":121,"score":0,"version":"Cambrian"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":122,"score":0,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":123,"score":0,"version":"Claude 3"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":124,"score":0,"version":"CodeLlama-7B"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":125,"score":0,"version":"Codestral"},{"company":"cohere","family":"Command","model":"command-r","place":126,"score":0,"version":"Command"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":127,"score":0,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":128,"score":0,"version":"DeepSeek R1"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":129,"score":0,"version":"Dream"},{"company":"baai","family":"Emu","model":"emu3-8b","place":130,"score":0,"version":"Emu3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":131,"score":0,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":132,"score":0,"version":"EXAONE Deep"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":133,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":134,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":135,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":136,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":137,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":138,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":139,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":140,"score":0,"version":"Falcon 3"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":141,"score":0,"version":"FastVLM"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":142,"score":0,"version":"Gemma 3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":143,"score":0,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo","place":144,"score":0,"version":"GPT-3.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":145,"score":0,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":146,"score":0,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4v","place":147,"score":0,"version":"GPT-4V"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":148,"score":0,"version":"GPT-5"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":149,"score":0,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":150,"score":0,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":151,"score":0,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":152,"score":0,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":153,"score":0,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":154,"score":0,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":155,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":156,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":157,"score":0,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":158,"score":0,"version":"Granite 4.0"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":159,"score":0,"version":"Grok 3"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":160,"score":0,"version":"InternVL 2"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":161,"score":0,"version":"InternVL 2"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":162,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":163,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":164,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":165,"score":0,"version":"InternVL 3.5"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":166,"score":0,"version":"Janus Pro"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":167,"score":0,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":168,"score":0,"version":"Kimi Linear"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":169,"score":0,"version":"LLaDA"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":170,"score":0,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-1b","place":171,"score":0,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":172,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":173,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":174,"score":0,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":175,"score":0,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":176,"score":0,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":177,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":178,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":179,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":180,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":181,"score":0,"version":"Llama 4"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":182,"score":0,"version":"Llama Nemotron v1"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":183,"score":0,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":184,"score":0,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":185,"score":0,"version":"LLaVA OneVision"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":186,"score":0,"version":"LLaVA OneVision"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":187,"score":0,"version":"Magistral 1.2"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":188,"score":0,"version":"MiniCPM V 2.0"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":189,"score":0,"version":"MiniMax M1"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":190,"score":0,"version":"Ministral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":191,"score":0,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":192,"score":0,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":193,"score":0,"version":"Mistral 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":194,"score":0,"version":"Mistral 3.2"},{"company":"apple","family":"MM1","model":"mm1-30b","place":195,"score":0,"version":"MM1"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":196,"score":0,"version":"MobileLLM"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":197,"score":0,"version":"Molmo"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":198,"score":0,"version":"MolmoE"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":199,"score":0,"version":"Moonlight"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":200,"score":0,"version":"Ouro"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":201,"score":0,"version":"Ouro"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":202,"score":0,"version":"PaliGemma"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":203,"score":0,"version":"Phi 3.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":204,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":205,"score":0,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b-instruct","place":206,"score":0,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":207,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":208,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":209,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":210,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":211,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":212,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":213,"score":0,"version":"Qwen2.5 Omni"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":214,"score":0,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":215,"score":0,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":216,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":217,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":218,"score":0,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":219,"score":0,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":220,"score":0,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":221,"score":0,"version":"Seed 1.5"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":222,"score":0,"version":"SmolLM1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":223,"score":0,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":224,"score":0,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":225,"score":0,"version":"SmolLM2"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":226,"score":0,"version":"xGen-MM"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-235B-A22B Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-235b-a22b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":100,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":197,"description":"","github":"","has_recent":0,"id":"faix-text","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score Text","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":85.71,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":63,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":58.81,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":4,"score":48.22,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":5,"score":47.03,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":6,"score":44.57,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":7,"score":44,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":8,"score":42.86,"version":"GPT-5.1"},{"company":"openai","family":"o3","model":"o3-low","place":9,"score":42.31,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":10,"score":42,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":11,"score":40,"version":"Seed 1.5"},{"company":"openai","family":"o1","model":"o1","place":12,"score":38.08,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":13,"score":35.43,"version":"Claude 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":14,"score":30.79,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":15,"score":30.33,"version":"Gemini 2.0"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":16,"score":30.12,"version":"ERNIE 4.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":17,"score":29.14,"version":"OLMo 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":18,"score":28.17,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":19,"score":28.17,"version":"Qwen3 Next"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":20,"score":27.44,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4","place":21,"score":27.43,"version":"GPT-4"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":22,"score":27.27,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":23,"score":24.56,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":24,"score":24.4,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":25,"score":24.4,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":26,"score":24.29,"version":"OLMo 3"},{"company":"openai","family":"o1","model":"o1-mini","place":27,"score":23.33,"version":"o1"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":28,"score":21.71,"version":"Grok 4"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":29,"score":21.5,"version":"Apriel Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":30,"score":21.12,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":31,"score":20.5,"version":"Qwen2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":32,"score":20,"version":"Gemini 2.0"},{"company":"mbzuai","family":"K2","model":"k2-think","place":33,"score":20,"version":"K2-THINK"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":34,"score":20,"version":"Ling 1T"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":35,"score":20,"version":"Mistral 3.2"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":36,"score":19.08,"version":"Llama TFree HAT"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":37,"score":18.75,"version":"Marin 32B"},{"company":"openai","family":"o3","model":"o3-mini-low","place":38,"score":18.17,"version":"o3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":39,"score":18.12,"version":"Llama 3.3"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":40,"score":18,"version":"dots.vlm1"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":41,"score":18,"version":"EXAONE 4.0"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":42,"score":18,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":43,"score":17.68,"version":"Llama 3.1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":44,"score":17.33,"version":"Claude 3.5"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":45,"score":17.25,"version":"Marin 32B"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":46,"score":17.2,"version":"Claude 4.5"},{"company":"google","family":"Gemma","model":"gemma-3","place":47,"score":16.49,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":48,"score":16.33,"version":"GPT-4"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":49,"score":16.29,"version":"OLMo 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":50,"score":16.21,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":51,"score":16,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":52,"score":15.92,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":53,"score":15.78,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":54,"score":15.67,"version":"Gemini 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":55,"score":15.47,"version":"DeepSeek R1"},{"company":"openai","family":"o4","model":"o4-mini","place":56,"score":15.2,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":57,"score":14.96,"version":"GPT-4.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":58,"score":14.57,"version":"Motif 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":59,"score":14.5,"version":"DeepSeek V2.5"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":60,"score":14.4,"version":"Grok Code"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":61,"score":14.07,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":62,"score":13.29,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":63,"score":13.17,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen-3-30b-a3b","place":64,"score":13.1,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":65,"score":13,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":66,"score":13,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":67,"score":12.95,"version":"Llama 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":68,"score":12.92,"version":"DeepSeek V3.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":69,"score":12.33,"version":"MiniMax M1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":70,"score":12.29,"version":"DeepSeek V3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":71,"score":12,"version":"GLM 4.6"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":72,"score":12,"version":"Granite 3.3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":73,"score":11.82,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":74,"score":11.79,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":75,"score":11.43,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":76,"score":11.43,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":77,"score":11.29,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":78,"score":11.29,"version":"Qwen3 VL"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":79,"score":11.29,"version":"QwQ"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":80,"score":11.23,"version":"OLMo 2"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":81,"score":11,"version":"Llama 3.1"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":82,"score":10.75,"version":"Mixtral"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":83,"score":10,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4.5","place":84,"score":9.78,"version":"GLM 4.5"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":85,"score":9.6,"version":"Llama 2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":86,"score":9.33,"version":"GPT-OSS"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":87,"score":9.23,"version":"Seed 1.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":88,"score":9.05,"version":"Falcon H1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":89,"score":9,"version":"MiniMax M1"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":90,"score":8.73,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen-3-coder-480b-a35b-instruct","place":91,"score":8.44,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":92,"score":8.36,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":93,"score":8.29,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":94,"score":7.83,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":95,"score":7.58,"version":"Gemini 1.5"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":96,"score":7,"version":"Granite 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b","place":97,"score":6.91,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":98,"score":6.77,"version":"GPT-4.5"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":99,"score":6.67,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":100,"score":6.55,"version":"Mistral 3.1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":101,"score":6.5,"version":"Nemotron Nano v2"},{"company":"cohere","family":"Command","model":"command-r","place":102,"score":6,"version":"Command"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":103,"score":5.78,"version":"Codestral"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":104,"score":5.76,"version":"SmollM3"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":105,"score":5.54,"version":"Granite 3.3"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":106,"score":5.33,"version":"Apriel 1.5"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":107,"score":5,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":108,"score":4.95,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":109,"score":4.86,"version":"Gemma 2"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":110,"score":4.84,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":111,"score":4.57,"version":"Llama 4"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":112,"score":4.51,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":113,"score":4,"version":"Claude 3.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":114,"score":4,"version":"Granite 4.0"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":115,"score":4,"version":"Grok 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":116,"score":4,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":117,"score":4,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":118,"score":4,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":119,"score":3.11,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":120,"score":3,"version":"Gemma 2"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":121,"score":2.91,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-3b-instruct","place":122,"score":2.88,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":123,"score":2.86,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":124,"score":2.86,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":125,"score":2.67,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":126,"score":2.63,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":127,"score":2.4,"version":"EXAONE Deep"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":128,"score":2.33,"version":"Llama 4"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":129,"score":2.29,"version":"LFM2"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":130,"score":2.11,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":131,"score":2,"version":"Gemma 2"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":132,"score":2,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":133,"score":2,"version":"Llama 4"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":134,"score":1.9,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":135,"score":1.85,"version":"Llama 2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":136,"score":1.78,"version":"Mistral"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":137,"score":1.71,"version":"LFM2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo","place":138,"score":1.6,"version":"GPT-3.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl","place":139,"score":1.6,"version":"MiMo VL"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":140,"score":1.5,"version":"Llama 3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":141,"score":1.45,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":142,"score":1.33,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":143,"score":1.03,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":144,"score":1,"version":"DeepSeek Coder"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":145,"score":0.86,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":146,"score":0.76,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":147,"score":0.67,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":148,"score":0.62,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":149,"score":0.62,"version":"Llama 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":150,"score":0.43,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":151,"score":0.42,"version":"Falcon 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":152,"score":0.4,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":153,"score":0.33,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":154,"score":0.29,"version":"Qwen3 VL"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":155,"score":0.21,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":156,"score":0.13,"version":"Llama 3.2"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":157,"score":0,"version":"Claude 3"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":158,"score":0,"version":"CodeLlama-7B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":159,"score":0,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":160,"score":0,"version":"DeepSeek R1"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":161,"score":0,"version":"Dream"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":162,"score":0,"version":"EXAONE 4.0"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":163,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":164,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":165,"score":0,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":166,"score":0,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":167,"score":0,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":168,"score":0,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":169,"score":0,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":170,"score":0,"version":"GPT-5"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":171,"score":0,"version":"Granite 3.2"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":172,"score":0,"version":"Grok 3"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":173,"score":0,"version":"InternVL"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":174,"score":0,"version":"LLaDA"},{"company":"meta","family":"Llama","model":"llama-1b","place":175,"score":0,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":176,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":177,"score":0,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":178,"score":0,"version":"Llama 3.2"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":179,"score":0,"version":"Llama Nemotron v1"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":180,"score":0,"version":"Magistral 1.2"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":181,"score":0,"version":"Magistral 1.2"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":182,"score":0,"version":"Ministral"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":183,"score":0,"version":"Mistral"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":184,"score":0,"version":"MobileLLM"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":185,"score":0,"version":"Moonlight"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":186,"score":0,"version":"Ouro"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":187,"score":0,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":188,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":189,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":190,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":191,"score":0,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":192,"score":0,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":193,"score":0,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":194,"score":0,"version":"SmolLM1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":195,"score":0,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":196,"score":0,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":197,"score":0,"version":"SmolLM2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":85.71,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":104,"description":"","github":"","has_recent":0,"id":"faix-visual","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score Visual","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":100,"version":"Gemini 3"},{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":2,"score":88.8,"version":"Holo1.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":3,"score":74,"version":"GPT-4.1"},{"company":"xai","family":"Grok","model":"grok-4-thinking","place":4,"score":70,"version":"Grok 4"},{"company":"openai","family":"o3","model":"o3-low","place":5,"score":66.67,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":66,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":7,"score":64.29,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":8,"score":52,"version":"Claude 4"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":9,"score":49.6,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":10,"score":48.8,"version":"Holo1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":11,"score":46.67,"version":"Gemini 2.0"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":12,"score":46.67,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":13,"score":45.09,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":14,"score":44.77,"version":"Gemini 2.5"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":15,"score":44.36,"version":"Molmo"},{"company":"openai","family":"GPT","model":"gpt-5","place":16,"score":42.87,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":17,"score":42.4,"version":"GPT-5.1"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":18,"score":40.52,"version":"dots.vlm1"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":19,"score":38.81,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":20,"score":31.67,"version":"Qwen2 VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":21,"score":28,"version":"UI-TARS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":22,"score":27.68,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":23,"score":26.67,"version":"Qwen3 VL"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":24,"score":26.55,"version":"Molmo"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":25,"score":24.4,"version":"UI-Venus"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":26,"score":24,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":27,"score":23.35,"version":"Qwen3 VL"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":28,"score":23.33,"version":"Llama 4"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":29,"score":22.35,"version":"ERNIE 4.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":30,"score":22,"version":"Holo1.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":31,"score":21.33,"version":"Claude 4.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl","place":32,"score":20.53,"version":"MiMo VL"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":33,"score":19.69,"version":"Claude 3.5"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":34,"score":18.67,"version":"Manzano"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":35,"score":18.22,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":36,"score":18.06,"version":"GPT-4o"},{"company":"stepfun","family":"Step","model":"step-3","place":37,"score":17,"version":"Step 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":38,"score":16.67,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":39,"score":15.72,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":40,"score":15.05,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b-instruct","place":41,"score":14.91,"version":"Qwen2 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":42,"score":14.8,"version":"InternVL 2"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":43,"score":14.5,"version":"GLM 4.1V"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":44,"score":12.73,"version":"FastVLM"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":45,"score":12.67,"version":"InternVL 3.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":46,"score":11.78,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":47,"score":11.56,"version":"Claude 4.1"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":48,"score":11.27,"version":"Llama 3.2"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":49,"score":10.8,"version":"MolmoE"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":50,"score":10.33,"version":"Gemini 1.5"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":51,"score":10.15,"version":"Phi-3"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":52,"score":9.67,"version":"Pixtral"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":53,"score":9.09,"version":"Claude 3"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":54,"score":9.09,"version":"Llama 3.2"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":55,"score":9,"version":"Bagel"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":56,"score":8.33,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":57,"score":8.1,"version":"Qwen2.5 VL"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":58,"score":8,"version":"Llama 4"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":59,"score":7.6,"version":"Holo1"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":60,"score":6.91,"version":"LLaVA OneVision"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":61,"score":6.64,"version":"Qwen3 VL"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":62,"score":6.18,"version":"FastVLM"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":63,"score":5.82,"version":"MM1.5"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":64,"score":5.67,"version":"Manzano"},{"company":"google","family":"Gemma","model":"gemma-3","place":65,"score":5.64,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":66,"score":5.1,"version":"InternVL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":67,"score":4.62,"version":"Gemini 2.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":68,"score":4,"version":"LLaVA OneVision"},{"company":"openai","family":"GPT","model":"gpt-5-nano-medium","place":69,"score":3.88,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking","place":70,"score":3.69,"version":"Kimi VL"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":71,"score":2.91,"version":"Cambrian"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":72,"score":2.91,"version":"InternVL 2.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":73,"score":2.5,"version":"Gemma 3"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":74,"score":2.4,"version":"Cambrian 1"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":75,"score":2.4,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":76,"score":2.15,"version":"Qwen2.5 Omni"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":77,"score":1.6,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":78,"score":0.94,"version":"Qwen2.5 VL"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":79,"score":0.8,"version":"Mistral 3.1"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":80,"score":0.8,"version":"SmolVLM2"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":81,"score":0.4,"version":"PaliGemma"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":82,"score":0.4,"version":"xGen-MM"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":83,"score":0.33,"version":"MM1.5"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":84,"score":0,"version":"BLIP-3O"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":85,"score":0,"version":"Cambrian 1"},{"company":"baai","family":"Emu","model":"emu3-8b","place":86,"score":0,"version":"Emu3"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":87,"score":0,"version":"FastVLM"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":88,"score":0,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-4v","place":89,"score":0,"version":"GPT-4V"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":90,"score":0,"version":"InternVL 2"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":91,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":92,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":93,"score":0,"version":"InternVL 3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":94,"score":0,"version":"InternVL 3.5"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":95,"score":0,"version":"Janus Pro"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":96,"score":0,"version":"LFM2-VL"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":97,"score":0,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":98,"score":0,"version":"LLaVA 1.5"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":99,"score":0,"version":"MiniCPM V 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":100,"score":0,"version":"Mistral 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":101,"score":0,"version":"Mistral 3.2"},{"company":"apple","family":"MM1","model":"mm1-30b","place":102,"score":0,"version":"MM1"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":103,"score":0,"version":"Phi 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":104,"score":0,"version":"Qwen2.5 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":100,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":22,"description":"","github":"","has_recent":0,"id":"faix-writing","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Meta Score Writing","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":1,"score":60,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":57.33,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen-3-30b-a3b","place":3,"score":54,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":4,"score":48,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":5,"score":36,"version":"DeepSeek V3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":6,"score":33.33,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":7,"score":30,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":29.33,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":9,"score":24,"version":"Claude 4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":10,"score":24,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b","place":11,"score":24,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":12,"score":16,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":13,"score":14,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":14,"score":12,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":15,"score":8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":16,"score":2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":17,"score":0,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":18,"score":0,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":19,"score":0,"version":"DeepSeek R1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":20,"score":0,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":21,"score":0,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":22,"score":0,"version":"Qwen3"}],"stars":"","top_company":"qwen","top_model":"Qwen3 235B A22B Instruct 2507","top_model_family":"Qwen","top_model_id":"qwen-3-235b-a22b-instruct-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":60,"topic":"Composite capability index","total_models":1613,"url":""},{"bench_models":2,"description":"Figure question answering benchmark evaluating visual reasoning over scientific figures and diagrams.","github":"","has_recent":0,"id":"figqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FigQA","paper":"","placements":[{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":1,"score":34,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":2,"score":29,"version":"Grok 4"}],"stars":"","top_company":"xai","top_model":"Grok 4.1 (Thinking)","top_model_family":"Grok","top_model_id":"grok-4.1-thinking","top_model_version":"Grok 4.1","top_open_source":0,"top_percent":34,"topic":"Figure understanding and QA","total_models":1613,"url":""},{"bench_models":5,"description":"Financial reasoning benchmark evaluating quantitative and qualitative finance problem solving.","github":"","has_recent":0,"id":"finance-reasoning","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FinanceReasoning","paper":"","placements":[{"company":"antgroup","family":"Ling","model":"ling-1t","place":1,"score":87.45,"version":"Ling 1T"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":86.65,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":3,"score":86.44,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":86.28,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":5,"score":84.83,"version":"Kimi K2"}],"stars":"","top_company":"antgroup","top_model":"Ling 1T","top_model_family":"Ling","top_model_id":"ling-1t","top_model_version":"Ling 1T","top_open_source":1,"top_percent":87.45,"topic":"Financial reasoning","total_models":1613,"url":""},{"bench_models":11,"description":"Interactive financial agent benchmark requiring multi-step tool use.","github":"","has_recent":0,"id":"financeagent","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FinanceAgent","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":60.7,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":60.7,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":3,"score":56.6,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":4,"score":56.6,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":55.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":55.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":7,"score":55.3,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":8,"score":54.7,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":9,"score":54.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":10,"score":54.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":11,"score":50.9,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5","place":12,"score":46.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":13,"score":44.5,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":14,"score":44.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":44.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":29.4,"version":"Gemini 2.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":60.7,"topic":"Agentic finance tasks","total_models":1613,"url":""},{"bench_models":6,"description":"Finance Agent v1.1 benchmark for interactive financial agent evaluation.","github":"","has_recent":0,"id":"financeagent-v1.1","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FinanceAgent v1.1","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":1,"score":63.3,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":60.1,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":59,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":58.8,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":55.2,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":54.5,"version":"Claude 4.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.6","top_model_family":"Claude","top_model_id":"claude-sonnet-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":63.3,"topic":"Agentic finance tasks","total_models":1613,"url":""},{"bench_models":4,"description":"FinanceBench full-document question answering benchmark requiring long-context financial understanding.","github":"","has_recent":0,"id":"financebench-fulldoc","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FinanceBench (FullDoc)","paper":"","placements":[{"company":"ai21","family":"Jamba","model":"jamba-mini-1.6","place":1,"score":45.4,"version":"Jamba Mini"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":2,"score":28.4,"version":"Llama 3.1"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":3,"score":19.2,"version":"Ministral"},{"company":"cohere","family":"Command","model":"command-r7b","place":4,"score":2.8,"version":"Command"}],"stars":"","top_company":"ai21","top_model":"Jamba Mini 1.6","top_model_family":"Jamba","top_model_id":"jamba-mini-1.6","top_model_version":"Jamba Mini","top_open_source":0,"top_percent":45.4,"topic":"Finance QA","total_models":1613,"url":""},{"bench_models":27,"description":"Financial search and comprehension benchmark measuring retrieval grounded reasoning over financial content.","github":"","has_recent":0,"id":"finsearchcomp","leaderboard":"https://randomtutu.github.io/FinSearchComp/","lower_is_better":false,"metric":"percent","name":"FinSearchComp","paper":"https://arxiv.org/pdf/2509.13160","placements":[{"company":"xai","family":"Grok","model":"grok-4","place":1,"score":68.9,"version":"Grok 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":2,"score":67.8,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":3,"score":66.2,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":4,"score":65.5,"version":"MiniMax M2"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":5,"score":63.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":6,"score":60.8,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":7,"score":59.1,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":49.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":9,"score":48.5,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":10,"score":47.4,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":11,"score":44,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":42.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":13,"score":42,"version":"Claude 4"},{"company":"bytedance","family":"Doubao","model":"doubao-1.5-vision-pro","place":14,"score":39.1,"version":"Doubao 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":15,"score":37.4,"version":"Qwen3"},{"company":"yuanbao","family":"YuanBao","model":"yuanbao-deepseek-v3","place":16,"score":30.5,"version":"YuanBao DeepSeek V3"},{"company":"yuanbao","family":"YuanBao","model":"yuanbao-hunyuan-t1-thinking","place":17,"score":29.8,"version":"YuanBao Hunyuan T1"},{"company":"yuanbao","family":"YuanBao","model":"yuanbao-deepseek-r1","place":18,"score":29.8,"version":"YuanBao DeepSeek R1"},{"company":"bytedance","family":"Doubao","model":"doubao-1.5-vision-pro-thinking","place":19,"score":29.8,"version":"Doubao 1.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":20,"score":29.5,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":21,"score":29.5,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":22,"score":29.2,"version":"GLM 4.6"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":23,"score":27,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":24,"score":26.2,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":25,"score":17.2,"version":"DeepSeek R1"},{"company":"baidu","family":"ERNIE","model":"ernie-x1","place":26,"score":16.6,"version":"ERNIE X1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":27,"score":10.4,"version":"Kimi K2"}],"stars":"","top_company":"xai","top_model":"Grok 4","top_model_family":"Grok","top_model_id":"grok-4","top_model_version":"Grok 4","top_open_source":0,"top_percent":68.9,"topic":"Financial retrieval","total_models":1613,"url":"https://randomtutu.github.io/FinSearchComp/"},{"bench_models":12,"description":"Chinese financial search and comprehension benchmark measuring retrieval-grounded reasoning over regional financial content.","github":"","has_recent":0,"id":"finsearchcomp-cn","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FinSearchComp-CN","paper":"","placements":[{"company":"bytedance","family":"Doubao","model":"doubao-1.5-vision-pro","place":1,"score":54.2,"version":"Doubao 1.5"},{"company":"yuanbao","family":"YuanBao","model":"yuanbao-deepseek-r1","place":2,"score":52.5,"version":"YuanBao DeepSeek R1"},{"company":"xai","family":"Grok","model":"grok-4","place":3,"score":51.9,"version":"Grok 4"},{"company":"yuanbao","family":"YuanBao","model":"yuanbao-hunyuan-t1-thinking","place":4,"score":50.5,"version":"YuanBao Hunyuan T1"},{"company":"bytedance","family":"Doubao","model":"doubao-1.5-vision-pro-thinking","place":5,"score":49,"version":"Doubao 1.5"},{"company":"yuanbao","family":"YuanBao","model":"yuanbao-deepseek-v3","place":6,"score":48.8,"version":"YuanBao DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":7,"score":46.4,"version":"GPT-5"},{"company":"baidu","family":"ERNIE","model":"ernie-x1","place":8,"score":40.8,"version":"ERNIE X1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":9,"score":40.5,"version":"DeepSeek R1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":10,"score":38.3,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":36.8,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":12,"score":21.9,"version":"Qwen3"}],"stars":"","top_company":"bytedance","top_model":"doubao-1-5-vision-pro","top_model_family":"Doubao","top_model_id":"doubao-1.5-vision-pro","top_model_version":"Doubao 1.5","top_open_source":0,"top_percent":54.2,"topic":"Financial retrieval (Chinese)","total_models":1613,"url":""},{"bench_models":5,"description":"Finance search competition tasks (tracks T2 and T3).","github":"","has_recent":0,"id":"finsearchcomp-t2t3","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FinSearchComp (T2\u0026T3)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":64.5,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":62.8,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":58.6,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":49.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":34,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":64.5,"topic":"Finance search","total_models":1613,"url":""},{"bench_models":7,"description":"Front-end React coding tasks and evaluation.","github":"","has_recent":0,"id":"flame-react-eval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Flame-React-Eval","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.6v","place":1,"score":86.3,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":2,"score":82.5,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":3,"score":78.8,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":4,"score":72.5,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":5,"score":63.8,"version":"Step 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":6,"score":36.3,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":7,"score":27.5,"version":"Gemma 3"}],"stars":"","top_company":"zai","top_model":"GLM-4.6V","top_model_family":"GLM","top_model_id":"glm-4.6v","top_model_version":"GLM 4.6V","top_open_source":1,"top_percent":86.3,"topic":"Frontend coding","total_models":1613,"url":""},{"bench_models":12,"description":"FLORES multilingual translation benchmark.","github":"https://github.com/facebookresearch/flores","has_recent":0,"id":"flores","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Flores","paper":"","placements":[{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":1,"score":88.88,"version":"EuroLLM"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":2,"score":88.81,"version":"Gemma 3"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":3,"score":88.17,"version":"Apertus"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":4,"score":88.15,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":5,"score":86.57,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":6,"score":86.36,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":7,"score":82.73,"version":"Mistral 3.2"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":8,"score":82.39,"version":"OLMo 2"},{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":9,"score":43.5,"version":"Aya"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":10,"score":38.9,"version":"Gemma 3"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":11,"score":32,"version":"Ministral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":12,"score":30.5,"version":"Qwen3"}],"stars":"","top_company":"eurollm","top_model":"EuroLLM-22B","top_model_family":"EuroLLM","top_model_id":"eurollm-22b","top_model_version":"EuroLLM","top_open_source":1,"top_percent":88.88,"topic":"Machine translation (multilingual)","total_models":1613,"url":"https://github.com/facebookresearch/flores"},{"bench_models":2,"description":"Fox Page benchmark evaluating OCR accuracy and layout understanding on Chinese document pages.","github":"","has_recent":0,"id":"fox-page-cn","leaderboard":"","lower_is_better":true,"metric":"percent","name":"Fox-Page-cn","paper":"","placements":[{"company":"bytedance","family":"Dolphin","model":"dolphin-1.5","place":1,"score":0.77,"version":"Dolphin"},{"company":"bytedance","family":"Dolphin","model":"dolphin","place":2,"score":1.31,"version":"Dolphin"}],"stars":"","top_company":"bytedance","top_model":"Dolphin 1.5","top_model_family":"Dolphin","top_model_id":"dolphin-1.5","top_model_version":"Dolphin","top_open_source":1,"top_percent":0.77,"topic":"Document OCR (Chinese)","total_models":1613,"url":""},{"bench_models":2,"description":"Fox Page benchmark evaluating OCR accuracy and layout understanding on English document pages.","github":"","has_recent":0,"id":"fox-page-en","leaderboard":"","lower_is_better":true,"metric":"percent","name":"Fox-Page-en","paper":"","placements":[{"company":"bytedance","family":"Dolphin","model":"dolphin-1.5","place":1,"score":0.74,"version":"Dolphin"},{"company":"bytedance","family":"Dolphin","model":"dolphin","place":2,"score":1.14,"version":"Dolphin"}],"stars":"","top_company":"bytedance","top_model":"Dolphin 1.5","top_model_family":"Dolphin","top_model_id":"dolphin-1.5","top_model_version":"Dolphin","top_open_source":1,"top_percent":0.74,"topic":"Document OCR (English)","total_models":1613,"url":""},{"bench_models":40,"description":"Frame-based interactive reasoning and dialogue benchmark.","github":"","has_recent":0,"id":"frames","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FRAMES","paper":"","placements":[{"company":"tongyi","family":"Tongyi","model":"tongyi-deepresearch","place":1,"score":90.6,"version":"Tongyi"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-235b","place":2,"score":88.3,"version":"MiroThinker v1.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":3,"score":87.4,"version":"GPT-4.1"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-30b","place":4,"score":87.3,"version":"MiroThinker v1.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":5,"score":87,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":86,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":86,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":8,"score":85,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":84.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":10,"score":84.59,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":11,"score":84.5,"version":"Seed 2.0"},{"company":"openai","family":"o3","model":"o3","place":12,"score":84,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":13,"score":84,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":14,"score":83.7,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":15,"score":83.7,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":16,"score":83.4,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":17,"score":82.9,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":18,"score":82.5,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":19,"score":81.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":20,"score":80.7,"version":"Claude 4"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":21,"score":80.5,"version":"Seed 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":22,"score":80.2,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":23,"score":79.2,"version":"DeepSeek V3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":24,"score":78.9,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-researcher","place":25,"score":78.8,"version":"Kimi Researcher"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":26,"score":78.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking-preview","place":27,"score":77.93,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":28,"score":77.1,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":29,"score":76.86,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":30,"score":76.3,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":31,"score":75.12,"version":"Qwen3"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1.5-30b-a3b","place":32,"score":74.76,"version":"QwenLong"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":33,"score":74.51,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":34,"score":73.54,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":35,"score":72.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":36,"score":72.69,"version":"GPT-OSS"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1","place":37,"score":72.39,"version":"QwenLong"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":38,"score":72,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":39,"score":70.27,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":40,"score":65.78,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":41,"score":58.1,"version":"Kimi K2"}],"stars":"","top_company":"tongyi","top_model":"Tongyi DeepResearch","top_model_family":"Tongyi","top_model_id":"tongyi-deepresearch","top_model_version":"Tongyi","top_open_source":0,"top_percent":90.6,"topic":"Interactive reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Question answering benchmark emphasizing up-to-date knowledge and recency.","github":"","has_recent":0,"id":"freshqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FreshQA","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking-2507","place":1,"score":66.89,"version":"Qwen3"},{"company":"menlo","family":"Jan","model":"jan-v1-2509","place":2,"score":55.07,"version":"Jan v1"},{"company":"menlo","family":"Jan","model":"jan-v1","place":3,"score":54.05,"version":"Jan v1"},{"company":"menlo","family":"Jan","model":"jan-v1-edge","place":4,"score":21.28,"version":"Jan v1"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":5,"score":17.57,"version":"Qwen3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-4B Thinking 2507","top_model_family":"Qwen","top_model_id":"qwen3-4b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":66.89,"topic":"Recency QA","total_models":1613,"url":""},{"bench_models":8,"description":"Frontier-level scientific reasoning and QA benchmark.","github":"","has_recent":0,"id":"frontierscience","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FrontierScience","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":25.2,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":17.5,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":3,"score":15.9,"version":"Grok 4"},{"company":"openai","family":"o3","model":"o3","place":4,"score":14.1,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":12.4,"version":"Gemini 3"},{"company":"openai","family":"o4","model":"o4-mini","place":6,"score":8.3,"version":"o4"},{"company":"openai","family":"o1","model":"o1","place":7,"score":3.2,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":8,"score":0.4,"version":"GPT-4o"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":25.2,"topic":"Science reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"Olympiad-level problems from the FrontierScience benchmark.","github":"","has_recent":0,"id":"frontierscience-olympiad","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FrontierScience Olympiad","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":75,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":2,"score":74,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":73,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":73,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":71,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":6,"score":70,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":7,"score":69,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":60,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":44,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":75,"topic":"Science reasoning (olympiad)","total_models":1613,"url":""},{"bench_models":9,"description":"Research-level problems from the FrontierScience benchmark.","github":"","has_recent":0,"id":"frontierscience-research","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FrontierScience Research","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":25,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":2,"score":25,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":21.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":4,"score":18.3,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":5,"score":18.3,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":16.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":15,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":11.7,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":3.3,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":25,"topic":"Science reasoning (research)","total_models":1613,"url":""},{"bench_models":6,"description":"Few-shot counting benchmark across 147 categories.","github":"","has_recent":0,"id":"fsc-147","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FSC-147","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":33.8,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":28,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":24.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":22,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":5,"score":17.9,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":12.1,"version":"Gemini 3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":33.8,"topic":"Few-shot counting","total_models":1613,"url":""},{"bench_models":46,"description":"End-to-end web/app development tasks and evaluation.","github":"","has_recent":0,"id":"fullstackbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FullStackBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":72.3,"version":"Claude 4.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":2,"score":71.4,"version":"IQuest Coder V1"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":3,"score":70.3,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.7","place":4,"score":70.2,"version":"GLM 4.7"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":5,"score":70.1,"version":"DeepSeek R1"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":6,"score":69.9,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":69.7,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":8,"score":69.4,"version":"DeepSeek R1"},{"company":"openai","family":"o3","model":"o3","place":9,"score":69.3,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":10,"score":68.5,"version":"GPT-4.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":11,"score":68.3,"version":"MiniMax M1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":12,"score":68.3,"version":"IQuest Coder V1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":13,"score":67.6,"version":"MiniMax M1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":14,"score":67.1,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":15,"score":66.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":16,"score":64.9,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":17,"score":64.7,"version":"DeepSeek V3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":18,"score":63.5,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":19,"score":62.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":20,"score":62.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":21,"score":62.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":22,"score":61.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":23,"score":60.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":24,"score":60.1,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":25,"score":58.9,"version":"GPT-OSS"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":26,"score":58.8,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":27,"score":58.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":28,"score":58.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":29,"score":57.4,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":30,"score":57.1,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":31,"score":56.55,"version":"Ling 1T"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-7b-instruct","place":32,"score":56.3,"version":"Hunyuan"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":33,"score":55.8,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":34,"score":55.7,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":35,"score":55.48,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":36,"score":54.4,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":37,"score":54,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":38,"score":53.1,"version":"Qwen2.5"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":39,"score":52.9,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":40,"score":51.5,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5","place":41,"score":50.92,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":42,"score":49.4,"version":"DeepSeek Coder V2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":43,"score":48.19,"version":"Gemini 2.5"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":44,"score":48,"version":"Nanbeige4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":45,"score":47.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":46,"score":42.2,"version":"Qwen2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":47,"score":38.6,"version":"Kimi Dev"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":48,"score":30.6,"version":"GPT-5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":72.3,"topic":"Full-stack development","total_models":1613,"url":""},{"bench_models":6,"description":"Chinese language full-stack development tasks and evaluation.","github":"","has_recent":0,"id":"fullstackbench-zh","leaderboard":"","lower_is_better":false,"metric":"percent","name":"FullStackBench (zh)","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":1,"score":63.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":2,"score":60.4,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":58.7,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":57.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":5,"score":55,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":35.2,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-235B-A22B","top_model_family":"Qwen","top_model_id":"qwen3-235b-a22b","top_model_version":"Qwen3","top_open_source":1,"top_percent":63.1,"topic":"Full-stack development","total_models":1613,"url":""},{"bench_models":30,"description":"Comprehensive benchmark for agentic tasks.","github":"https://github.com/GAIA-benchmark/GAIA","has_recent":0,"id":"gaia","leaderboard":"https://gaia-benchmark.github.io/leaderboard/","lower_is_better":false,"metric":"percent","name":"GAIA","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":87.4,"version":"Seed 1.8"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.0-72b","place":2,"score":81.9,"version":"MiroThinker v1.0"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-235b","place":3,"score":80.8,"version":"MiroThinker v1.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":79.9,"version":"Kimi K2.5"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":5,"score":77.4,"version":"Intern-S1"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":6,"score":76.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":7,"score":76.4,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":8,"score":75.7,"version":"MiniMax M2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":75.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":74.8,"version":"Gemini 3"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-30b","place":11,"score":72,"version":"MiroThinker v1.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":12,"score":71.9,"version":"GLM 4.6"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":13,"score":71.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":14,"score":71.1,"version":"GPT-5.2"},{"company":"tongyi","family":"Tongyi","model":"tongyi-deepresearch","place":15,"score":70.9,"version":"Tongyi"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":16,"score":68.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":17,"score":68.3,"version":"Claude 4"},{"company":"openai","family":"DeepResearch","model":"openai-deepresearch","place":18,"score":67.4,"version":"DeepResearch"},{"company":"zai","family":"GLM","model":"glm-4.5","place":19,"score":66,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":20,"score":66,"version":"Claude 4.5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v0.2-32b","place":21,"score":64.1,"version":"MiroThinker v0.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":22,"score":63.5,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":23,"score":63.1,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":24,"score":60.2,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":25,"score":60.2,"version":"Kimi K2"},{"company":"openai","family":"o4","model":"o4-mini","place":26,"score":60,"version":"o4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":27,"score":57.7,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":28,"score":57.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":29,"score":47.8,"version":"Qwen3 VL"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":30,"score":33.9,"version":"Youtu-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":31,"score":25.5,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":32,"score":11.7,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":33,"score":11.4,"version":"Qwen3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":87.4,"topic":"General AI tasks","total_models":1613,"url":"https://github.com/GAIA-benchmark/GAIA"},{"bench_models":7,"description":"GAIA benchmark subset without file inputs.","github":"https://github.com/GAIA-benchmark/GAIA","has_recent":0,"id":"gaia-no-file","leaderboard":"https://gaia-benchmark.github.io/leaderboard/","lower_is_better":false,"metric":"percent","name":"GAIA (no file)","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":1,"score":84.5,"version":"Step 3.5 Flash"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":2,"score":78.2,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":75.9,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":4,"score":75.6,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":5,"score":75.1,"version":"DeepSeek V3.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":6,"score":64.3,"version":"MiniMax M2.1"},{"company":"zai","family":"GLM","model":"glm-4.7","place":7,"score":61.9,"version":"GLM 4.7"}],"stars":"","top_company":"stepfun","top_model":"Step-3.5 Flash 20260204","top_model_family":"Step","top_model_id":"step-3.5-flash-20260204","top_model_version":"Step 3.5 Flash","top_open_source":1,"top_percent":84.5,"topic":"General AI tasks","total_models":1613,"url":"https://github.com/GAIA-benchmark/GAIA"},{"bench_models":14,"description":"Grounded agentic intelligence benchmark version 2 covering multi-tool tasks.","github":"","has_recent":0,"id":"gaia2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GAIA 2","paper":"","placements":[{"company":"antgroup","family":"Ring","model":"ring-1t-2.5","place":1,"score":75,"version":"Ring 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":2,"score":68.96,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":3,"score":55.21,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":42.1,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":5,"score":34.8,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-low","place":6,"score":34.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":25.8,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":8,"score":20.1,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":9,"score":18.2,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4","place":10,"score":12.9,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":11,"score":11.6,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":12,"score":7.4,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":13,"score":7.4,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":14,"score":4.4,"version":"Llama 3.3"}],"stars":"","top_company":"antgroup","top_model":"Ring-1T-2.5","top_model_family":"Ring","top_model_id":"ring-1t-2.5","top_model_version":"Ring 2.5","top_open_source":1,"top_percent":75,"topic":"General agent tasks","total_models":1613,"url":""},{"bench_models":4,"description":"GAOKAO benchmark measuring Chinese college entrance exam performance.","github":"","has_recent":0,"id":"gaokao-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GAOKAO-Bench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":1,"score":94.53,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":2,"score":93.9,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":3,"score":93.24,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":4,"score":86.12,"version":"LLaDA 2.0"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Instruct-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-instruct-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":94.53,"topic":"Chinese exams","total_models":1613,"url":""},{"bench_models":17,"description":"GDPVal benchmark evaluating broad general capabilities of LLMs across diverse tasks.","github":"","has_recent":0,"id":"gdpval","leaderboard":"https://openai.com/index/gdpval/","lower_is_better":false,"metric":"percent","name":"GDPVal","paper":"https://cdn.openai.com/pdf/d5eb7428-c4e9-4a33-bd86-86dd4bcf12ce/GDPval.pdf","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":73.5,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":2,"score":70.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":3,"score":70.9,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":4,"score":70.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":5,"score":70.9,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":70.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":61.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":59.6,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":9,"score":59,"version":"MiniMax M2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":54.5,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":11,"score":53.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":12,"score":47.6,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":13,"score":38.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":14,"score":38.8,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-high","place":15,"score":34.1,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":28.1,"version":"Gemini 3"},{"company":"openai","family":"o4","model":"o4-mini-high","place":17,"score":27.9,"version":"o4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":25.5,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":19,"score":24.6,"version":"MiniMax M2.1"},{"company":"xai","family":"Grok","model":"grok-4","place":20,"score":24.3,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":21,"score":12.4,"version":"GPT-4o"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":73.5,"topic":"General capability","total_models":1613,"url":"https://huggingface.co/datasets/openai/gdpval"},{"bench_models":8,"description":"GDPVal Artificial Analysis Elo rating for office-style tasks.","github":"","has_recent":0,"id":"gdpval-aa-elo","leaderboard":"","lower_is_better":false,"metric":"elo","name":"GDPVal-AA Elo","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":1,"score":1633,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":1606,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":3,"score":1606,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":1462,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":1462,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":1416,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":1416,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":8,"score":1317,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":9,"score":1277,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":10,"score":1277,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":1276,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":1201,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":1195,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":14,"score":1195,"version":"Gemini 3"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.6","top_model_family":"Claude","top_model_id":"claude-sonnet-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":1633,"topic":"Office tasks","total_models":1613,"url":"https://openai.com/index/gdpval/"},{"bench_models":18,"description":"General tool-use benchmark covering web and API tasks.","github":"","has_recent":0,"id":"general-tool-use","leaderboard":"","lower_is_better":false,"metric":"percent","name":"General Tool Use","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":78.9,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":2,"score":78.2,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":77.7,"version":"Claude 4.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":4,"score":73.9,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":5,"score":71.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":6,"score":71.2,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":7,"score":70.3,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":8,"score":68.8,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":9,"score":68.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":10,"score":64.8,"version":"GLM 4.7"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":11,"score":64.7,"version":"KAT"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":12,"score":64.4,"version":"GPT-5.1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":13,"score":63.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":14,"score":62.3,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":15,"score":59.9,"version":"Qwen2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":16,"score":55.5,"version":"Kimi Dev"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":17,"score":54.2,"version":"Qwen2.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":18,"score":51.7,"version":"IQuest Coder V1"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":78.9,"topic":"Tool use","total_models":1613,"url":""},{"bench_models":5,"description":"Geospatial visual QA and reasoning (set 1).","github":"","has_recent":0,"id":"geobench1","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GeoBench1","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.5v","place":1,"score":79.7,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":2,"score":76,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":3,"score":72.9,"version":"Step 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":4,"score":57.5,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":5,"score":48,"version":"Kimi VL"}],"stars":"","top_company":"zai","top_model":"GLM-4.5V","top_model_family":"GLM","top_model_id":"glm-4.5v","top_model_version":"GLM 4.5V","top_open_source":1,"top_percent":79.7,"topic":"Geospatial reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Global multilingual grade school math word problems.","github":"","has_recent":0,"id":"global-mgsm","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GlobalMGSM","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":1,"score":60.9,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":2,"score":55.4,"version":"Gemma 3"},{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":3,"score":52.8,"version":"Aya"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":4,"score":49.6,"version":"Ministral 3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-4B","top_model_family":"Qwen","top_model_id":"qwen3-4b","top_model_version":"Qwen3","top_open_source":1,"top_percent":60.9,"topic":"Math (multilingual)","total_models":1613,"url":""},{"bench_models":11,"description":"Full Global-MMLU evaluation across diverse languages and regions.","github":"","has_recent":0,"id":"global-mmlu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Global-MMLU","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":1,"score":82,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":2,"score":81.9,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":3,"score":80.7,"version":"Kimi K2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":4,"score":77.8,"version":"Llama 3.3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":5,"score":76.6,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":6,"score":65.1,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":7,"score":53.5,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":8,"score":50.54,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":9,"score":49.5,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":10,"score":46.8,"version":"Llama 3.1"},{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":11,"score":44.9,"version":"Aya"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.2-Exp","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.2-exp","top_model_version":"DeepSeek V3.2","top_open_source":1,"top_percent":82,"topic":"Multi-domain knowledge (global)","total_models":1613,"url":""},{"bench_models":14,"description":"Lightweight global variant of MMLU covering diverse languages and regions.","github":"","has_recent":0,"id":"global-mmlu-lite","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Global-MMLU-Lite","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":89.2,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":2,"score":86.5,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":3,"score":83.4,"version":"Gemini 2.0"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":4,"score":83,"version":"Mi:dm K 2.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":5,"score":80.8,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":6,"score":75.1,"version":"Gemma 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":7,"score":74.47,"version":"Nemotron 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":8,"score":73.7,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":9,"score":69.5,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":10,"score":68.6,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":11,"score":64.8,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":12,"score":54.5,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":13,"score":41.9,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":14,"score":34.2,"version":"Gemma 3"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":89.2,"topic":"Multi-domain knowledge (global)","total_models":1613,"url":"https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite"},{"bench_models":27,"description":"Physical commonsense reasoning benchmark spanning 100 languages and diverse cultural contexts.","github":"","has_recent":0,"id":"global-piqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Global PIQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-flash","place":1,"score":95.6,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":95,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":93.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":93.9,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":93.4,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":6,"score":93.2,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":93.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":92.8,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":9,"score":92.3,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":10,"score":92.1,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":11,"score":91.6,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":12,"score":91.6,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":91.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":14,"score":91.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":15,"score":90.9,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":16,"score":90.9,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":17,"score":90.2,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":18,"score":90.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":19,"score":90.1,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":20,"score":89.8,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":21,"score":89.3,"version":"Kimi K2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":22,"score":89.2,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":23,"score":88.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":24,"score":88.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":25,"score":87.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":26,"score":86.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":27,"score":86,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":28,"score":85.7,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":29,"score":85.6,"version":"Grok 4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":30,"score":84.1,"version":"GPT-OSS"},{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":31,"score":68.3,"version":"Aya"}],"stars":"","top_company":"google","top_model":"Gemini 3 Flash","top_model_family":"Gemini","top_model_id":"gemini-3-flash","top_model_version":"Gemini 3","top_open_source":0,"top_percent":95.6,"topic":"Commonsense reasoning across 100 Languages and Cultures","total_models":1613,"url":""},{"bench_models":5,"description":"Gorilla API Bench tool-use evaluation.","github":"","has_recent":0,"id":"gorilla-api-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Gorilla Benchmark API Bench","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":1,"score":35.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":2,"score":29.7,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":3,"score":14.7,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":4,"score":8.2,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":5,"score":1.7,"version":"Llama 3"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B","top_model_family":"Llama","top_model_id":"llama-3.1-405b","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":35.3,"topic":"Tool use","total_models":1613,"url":"https://gorilla.cs.berkeley.edu/benchmark"},{"bench_models":150,"description":"Graduate-level question answering evaluating advanced reasoning.","github":"https://github.com/idavidrein/gpqa","has_recent":0,"id":"gpqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GPQA","paper":"https://arxiv.org/abs/2311.12022","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":1,"score":92.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":92.4,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":91.9,"version":"Gemini 3"},{"company":"xai","family":"Grok","model":"grok-4","place":4,"score":88.4,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":5,"score":88.4,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":87.6,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":7,"score":87.4,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":87,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":9,"score":85.1,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":10,"score":83.4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":11,"score":82.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":12,"score":82.8,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":13,"score":82.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":14,"score":81.4,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":15,"score":81.1,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":16,"score":81,"version":"DeepSeek R1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":17,"score":81,"version":"GLM 4.6"},{"company":"zai","family":"GLM","model":"glm-4.5","place":18,"score":79.9,"version":"GLM 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":19,"score":79.9,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":20,"score":77.7,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":21,"score":77.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":22,"score":77.5,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":23,"score":77.2,"version":"Qwen3 Next"},{"company":"zai","family":"GLM","model":"glm-4.5","place":24,"score":77,"version":"GLM 4.5"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":25,"score":75.25,"version":"Ring 2.0"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":26,"score":75,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":27,"score":74.4,"version":"Qwen3 VL"},{"company":"primeintellect","family":"INTELLECT","model":"intellect-3","place":28,"score":74.4,"version":"INTELLECT"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":29,"score":74.2,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":30,"score":73.4,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":31,"score":73.3,"version":"GLM 4.5"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":32,"score":73.1,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":33,"score":73.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":34,"score":73.1,"version":"Qwen3"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":35,"score":73.1,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":36,"score":73,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":37,"score":72.9,"version":"Qwen3 Next"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":38,"score":72.85,"version":"Ring 2.0"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":39,"score":72.78,"version":"dots.vlm1"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":40,"score":71.6,"version":"Nemotron OpenReasoning"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":41,"score":71.5,"version":"GPT-OSS"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":42,"score":71.4,"version":"Seed OSS"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":43,"score":71.4,"version":"GPT-4.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":44,"score":71.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":45,"score":70.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":46,"score":70.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":47,"score":70.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":48,"score":70.4,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":49,"score":70,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":50,"score":69.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":51,"score":69.6,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":52,"score":69.16,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":53,"score":68.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":54,"score":68.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":55,"score":67.4,"version":"Qwen3 VL"},{"company":"antgroup","family":"Ring","model":"ring-mini-sparse-2.0-exp","place":56,"score":67.3,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":57,"score":67.3,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":58,"score":67.3,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":59,"score":67.12,"version":"Ling 2.0"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":60,"score":66.9,"version":"GPT-4o"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":61,"score":66.67,"version":"LLaDA 2.1"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":62,"score":64,"version":"Nemotron Nano v2"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":63,"score":62.9,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":64,"score":62.31,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":65,"score":61.98,"version":"LLaDA 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":66,"score":61.8,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":67,"score":61.1,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":68,"score":59.6,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":69,"score":59.41,"version":"Ling 2.0"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":70,"score":58.1,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":71,"score":57.6,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":72,"score":57.34,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking","place":73,"score":55.9,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":74,"score":55.7,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":75,"score":54.6,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":76,"score":54.5,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":77,"score":54.14,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":78,"score":53.28,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl-thinking","place":79,"score":51.2,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":80,"score":50.7,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":81,"score":49.49,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":82,"score":48.36,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":83,"score":48.01,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":84,"score":47.76,"version":"LLaDA 2.0"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":85,"score":46.7,"version":"Llama 3.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":86,"score":46.59,"version":"LLaDA 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":87,"score":45,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":88,"score":44.6,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":89,"score":43.94,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":90,"score":42.18,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":91,"score":41.7,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":92,"score":41.53,"version":"Falcon H1"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b-exp","place":93,"score":41.31,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl","place":94,"score":40.8,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":95,"score":40.63,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":96,"score":39.9,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":97,"score":39.9,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":98,"score":39.5,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":99,"score":39,"version":"Qwen2.5"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":100,"score":38.89,"version":"LFM2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":101,"score":37.67,"version":"Qwen2.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":102,"score":36.6,"version":"Nemotron Nano v2"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":103,"score":36.49,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":104,"score":36.4,"version":"OLMo 2"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":105,"score":36.33,"version":"Falcon H1"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":106,"score":36,"version":"Llama TFree HAT"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":107,"score":36,"version":"Llama TFree HAT"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":108,"score":35.7,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4","place":109,"score":35.7,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":110,"score":34.85,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":111,"score":34.85,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":112,"score":34.6,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":113,"score":34.31,"version":"Qwen2.5"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":114,"score":34,"version":"Marin 32B"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":115,"score":33.98,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":116,"score":33.89,"version":"Falcon H1"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":117,"score":33.84,"version":"LFM2"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":118,"score":33.39,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":119,"score":33.22,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":120,"score":32.72,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":121,"score":32.59,"version":"Granite 4.0"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":122,"score":32.2,"version":"OLMo 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":123,"score":32.15,"version":"Granite 4.0"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":124,"score":32.1,"version":"Marin 32B"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":125,"score":31.99,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":126,"score":31.8,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":127,"score":31.79,"version":"Qwen2.5"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":128,"score":31.6,"version":"Nemotron OpenReasoning"},{"company":"liquidai","family":"LFM","model":"lfm2-1.2b","place":129,"score":31.47,"version":"LFM2"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":130,"score":31.21,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-it","place":131,"score":31.11,"version":"Gemma 3n"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":132,"score":30.6,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":133,"score":30.4,"version":"Llama 3.1"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-a0.6b","place":134,"score":30.2,"version":"Aquif 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":135,"score":30.2,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":136,"score":30.14,"version":"Granite 4.0"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":137,"score":29.95,"version":"Falcon H1"},{"company":"nvidia","family":"Nemotron Flash","model":"nemotron-flash-3b","place":138,"score":29.54,"version":"Nemotron Flash"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":139,"score":29.51,"version":"Gemma 3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":140,"score":29.29,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":141,"score":29.19,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":142,"score":28.99,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":143,"score":28.94,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":144,"score":28.84,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":145,"score":28.69,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":146,"score":28.69,"version":"Qwen2.5"},{"company":"aquif","family":"Aquif","model":"aquif-3-0.4b","place":147,"score":28.5,"version":"Aquif 3"},{"company":"liquidai","family":"LFM","model":"lfm2-700m","place":148,"score":28.48,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":149,"score":28.28,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":150,"score":28.02,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":151,"score":27.72,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":152,"score":27.46,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":153,"score":26.85,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":154,"score":26.76,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":155,"score":26.59,"version":"Llama 3.2"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":156,"score":26.57,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":157,"score":26.46,"version":"Granite 4.0"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":158,"score":26.34,"version":"Falcon H1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":159,"score":26.31,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":160,"score":26.26,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":161,"score":25.84,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":162,"score":24.24,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":163,"score":24.24,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":164,"score":22.14,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":165,"score":22.1,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":166,"score":21.72,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":167,"score":21.07,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":168,"score":16.57,"version":"Llama 3"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":169,"score":11.34,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":170,"score":9.3,"version":"SmolLM2"},{"company":"arcee","family":"Llama Spark","model":"llama-spark-8b","place":171,"score":6.6,"version":"Llama Spark 8B"},{"company":"tii","family":"Falcon","model":"falcon-e-1b-base","place":172,"score":2.8,"version":"Falcon E"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":173,"score":0,"version":"SmolLM2"}],"stars":"406","top_company":"openai","top_model":"GPT-5.2 Thinking","top_model_family":"GPT","top_model_id":"gpt-5.2-thinking","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":92.4,"topic":"Graduate-level QA","total_models":1613,"url":"https://gpqa.github.io/"},{"bench_models":294,"description":"Hard subset of GPQA (diamond level).","github":"","has_recent":0,"id":"gpqa-diamond","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GPQA-diamond","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":94.3,"version":"Gemini 3.1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":93.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":93.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":4,"score":92.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":5,"score":92.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":6,"score":92.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":7,"score":92.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":8,"score":92.4,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":91.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":91.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":11,"score":91.3,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":12,"score":91.3,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":91,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":14,"score":90.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":15,"score":90.4,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":16,"score":90,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":17,"score":89.9,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5-pro","place":18,"score":89.4,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":19,"score":88.9,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":20,"score":88.1,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":21,"score":88.1,"version":"GPT-5.1"},{"company":"xai","family":"Grok","model":"grok-4","place":22,"score":87.7,"version":"Grok 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":23,"score":87.6,"version":"Kimi K2.5"},{"company":"xai","family":"Grok","model":"grok-4","place":24,"score":87.5,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":25,"score":87,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":26,"score":87,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":27,"score":87,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":28,"score":86.9,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":29,"score":86.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":30,"score":86.9,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":31,"score":86.6,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":32,"score":86.4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":33,"score":85.7,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":34,"score":85.7,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":35,"score":85.7,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":36,"score":85.7,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":37,"score":85.7,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":38,"score":85.5,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":39,"score":85.4,"version":"Kimi K2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":40,"score":85.2,"version":"LongCat-Flash"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":41,"score":85.1,"version":"Seed 2.0"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":42,"score":84.9,"version":"GLM 4.7"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":43,"score":84.7,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":44,"score":84.51,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":45,"score":84.5,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":46,"score":84.4,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":47,"score":84.3,"version":"Grok 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":48,"score":84.3,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":49,"score":84.2,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":50,"score":84,"version":"DeepSeek V3.2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":51,"score":83.8,"version":"Seed 1.8"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":52,"score":83.8,"version":"Nanbeige4.1"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":53,"score":83.7,"version":"MiMo V2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":54,"score":83.7,"version":"MiMo V2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":55,"score":83.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":56,"score":83.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":57,"score":83.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":58,"score":83.4,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3","place":59,"score":83.3,"version":"o3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":60,"score":83.2,"version":"openPangu-R"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":61,"score":83,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":62,"score":83,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":63,"score":82.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":64,"score":82.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":65,"score":82.8,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":66,"score":82.7,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":67,"score":82.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":68,"score":82.4,"version":"DeepSeek V3.2"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":69,"score":82.2,"version":"Nanbeige4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":70,"score":82.1,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-high","place":71,"score":81.3,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":72,"score":81.3,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":73,"score":81.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":74,"score":81.1,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0528","place":75,"score":81,"version":"DeepSeek V3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":76,"score":81,"version":"GLM 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":77,"score":81,"version":"Claude 4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":78,"score":81,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":79,"score":80.9,"version":"Claude 4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":80,"score":80.7,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":81,"score":80.7,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":82,"score":80.52,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":83,"score":80.5,"version":"Qwen3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":84,"score":80.5,"version":"LongCat-Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":85,"score":80.1,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":86,"score":80.1,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":87,"score":80.1,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":88,"score":79.9,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.5","place":89,"score":79.9,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":90,"score":79.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":91,"score":79.6,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":92,"score":79.46,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":93,"score":79.2,"version":"DeepSeek V3.1"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":94,"score":79.1,"version":"K-EXAONE"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":95,"score":79,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":96,"score":79,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":97,"score":79,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":98,"score":78.2,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5","place":99,"score":78.2,"version":"GLM 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":100,"score":78,"version":"MiniMax M2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":101,"score":78,"version":"GPT-OSS"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t2-chimera","place":102,"score":77.9,"version":"DeepSeek-TNG R1T2"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":103,"score":77.72,"version":"Cogito 671B v2.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":104,"score":77.7,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":105,"score":77.7,"version":"Claude 4"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":106,"score":77.7,"version":"MiniMax M2"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":107,"score":77.3,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":108,"score":77.15,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":109,"score":77.11,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":110,"score":77.04,"version":"GPT-OSS"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":111,"score":76.8,"version":"o3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":112,"score":76.8,"version":"openPangu-R"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":113,"score":76.7,"version":"Kimi K2"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":114,"score":76.26,"version":"Magistral 1.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":115,"score":76.2,"version":"MiniMax M2"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-ultra-253b-v1","place":116,"score":76,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":117,"score":75.9,"version":"Qwen3 Next"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":118,"score":75.8,"version":"GLM 4.5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":119,"score":75.57,"version":"Ling 2.5"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":120,"score":75.4,"version":"EXAONE 4.0"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":121,"score":75.3,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":122,"score":75.2,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":123,"score":75.1,"version":"Kimi K2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":124,"score":74.98,"version":"Nemotron"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":125,"score":74.9,"version":"Claude 4"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":126,"score":74.8,"version":"Llama Nemotron v1.5"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":127,"score":74.43,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":128,"score":74.33,"version":"Qwen3 Next"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":129,"score":74.2,"version":"Kimi K2"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":130,"score":73.9,"version":"Magistral 1.2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":131,"score":73.9,"version":"EXAONE 4.0"},{"company":"meta","family":"Llama","model":"llama-4-behemoth","place":132,"score":73.7,"version":"Llama 4"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":133,"score":73.48,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":134,"score":73.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":135,"score":73.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":136,"score":73.4,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":137,"score":73.3,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":138,"score":73,"version":"Claude 4.5"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t-chimera","place":139,"score":72,"version":"DeepSeek-TNG R1T"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":140,"score":71.7,"version":"Kimi Linear"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":141,"score":71.5,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0120","place":142,"score":71.5,"version":"0120"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":143,"score":71.5,"version":"GPT-OSS"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.1","place":144,"score":71.46,"version":"Magistral 1.1"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":145,"score":71.4,"version":"GPT-4.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":146,"score":71.3,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":147,"score":71.1,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":148,"score":71.08,"version":"DeepSeek R1"},{"company":"mbzuai","family":"K2","model":"k2-think","place":149,"score":71.08,"version":"K2-THINK"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.0","place":150,"score":70.83,"version":"Magistral 1.0"},{"company":"magistral","family":"Magistral","model":"magistral-medium","place":151,"score":70.8,"version":"Magistral"},{"company":"mistral","family":"Magistral","model":"magistral-medium-2506","place":152,"score":70.8,"version":"Magistral 2506"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":153,"score":70.7,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":154,"score":70.61,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":155,"score":70.6,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":156,"score":70.56,"version":"TeleChat3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":157,"score":70.2,"version":"Gemini 2.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":158,"score":70.07,"version":"Magistral 1.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":159,"score":70,"version":"MiniMax M1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":160,"score":70,"version":"Claude 4"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":161,"score":70,"version":"Motif 2"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":162,"score":70,"version":"Mi:dm K 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":163,"score":69.89,"version":"Kimi Linear"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":164,"score":69.8,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":165,"score":69.8,"version":"Llama 4"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":166,"score":69.6,"version":"Nemotron Cascade"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":167,"score":69.4,"version":"GPT-OSS"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":168,"score":69.3,"version":"K2 V2"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":169,"score":69.2,"version":"MiniMax M1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1","place":170,"score":69.2,"version":"MiniMax M1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":171,"score":68.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":172,"score":68.69,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":173,"score":68.4,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":174,"score":68.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":175,"score":68.4,"version":"Qwen3"},{"company":"antgroup","family":"Ring","model":"ring-mini-2.0","place":176,"score":68.24,"version":"Ring 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":177,"score":68.2,"version":"Gemini 2.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-2506","place":178,"score":68.2,"version":"Magistral 2506"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":179,"score":68.2,"version":"Apriel 1.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.0","place":180,"score":68.18,"version":"Magistral 1.0"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":181,"score":68.18,"version":"HY"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":182,"score":68.1,"version":"Solar Open"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":183,"score":67.9,"version":"Phi-4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":184,"score":67.68,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":185,"score":67.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":186,"score":67.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":187,"score":67.2,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":188,"score":67.1,"version":"Llama 4"},{"company":"microsoft","family":"Phi","model":"phi-4-reasoning","place":189,"score":67.1,"version":"Phi 4"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":190,"score":66.78,"version":"LongCat-Flash-Lite"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":191,"score":66.7,"version":"QwQ"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":192,"score":66.7,"version":"Qwen3"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":193,"score":66.7,"version":"Llama Nemotron v1"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":194,"score":66.7,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":195,"score":66.5,"version":"Nemotron Cascade"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":196,"score":66.3,"version":"Magistral 1.2"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":197,"score":66.3,"version":"GPT-4.1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":198,"score":66.24,"version":"QwQ"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":199,"score":66,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":200,"score":66,"version":"GPT-OSS"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":201,"score":66,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":202,"score":65.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":203,"score":65.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":204,"score":65.8,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.1","place":205,"score":65.78,"version":"Magistral 1.1"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":206,"score":65.56,"version":"HY"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":207,"score":65.53,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":208,"score":65.45,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":209,"score":65.4,"version":"GPT-OSS"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":210,"score":65.2,"version":"EXAONE 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-70b","place":211,"score":65.2,"version":"DeepSeek R1"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":212,"score":65.15,"version":"HY"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":213,"score":64.7,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":214,"score":64,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":215,"score":64,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":216,"score":64,"version":"Nemotron Nano v2"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":217,"score":63.6,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":218,"score":63.38,"version":"Qwen3"},{"company":"arcee","family":"Trinity","model":"trinity-large-preview","place":219,"score":63.3,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":220,"score":62.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":221,"score":62.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":222,"score":62,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":223,"score":62,"version":"Qwen3"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":224,"score":62,"version":"HyperCLOVA X"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":225,"score":61.7,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":226,"score":61.4,"version":"DeepSeek R1"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":227,"score":61.3,"version":"Falcon H1R"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":228,"score":61.3,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":229,"score":61.2,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":230,"score":61.2,"version":"GPT-OSS"},{"company":"antgroup","family":"Ring","model":"ring-lite","place":231,"score":61.1,"version":"Ring Lite"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":232,"score":60.91,"version":"OLMo 3"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":233,"score":60.6,"version":"K2 V2"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":234,"score":60.1,"version":"Gemini 2.0"},{"company":"openai","family":"o1","model":"o1-mini","place":235,"score":60,"version":"o1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":236,"score":59.8,"version":"OLMo 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":237,"score":59.1,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":238,"score":58.59,"version":"Qwen3"},{"company":"arcee","family":"Trinity","model":"trinity-mini","place":239,"score":58.55,"version":"Trinity"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":240,"score":58.3,"version":"MiMo VL"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":241,"score":58,"version":"OLMo 3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":242,"score":57.8,"version":"Kanana 2"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":243,"score":57.4,"version":"Apriel Nemotron"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":244,"score":57.2,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":245,"score":57.2,"version":"Llama 4"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":246,"score":57.1,"version":"Mistral 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":247,"score":56.9,"version":"Falcon H1"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":248,"score":56.8,"version":"Nemotron-H"},{"company":"aquif","family":"Aquif","model":"aquif-3-moe-17b-a2.8b","place":249,"score":56.7,"version":"Aquif 3"},{"company":"mistral","family":"Magistral","model":"magistral-small-2509","place":250,"score":56.5,"version":"Magistral 1.2"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":251,"score":56.3,"version":"MiMo VL"},{"company":"microsoft","family":"Phi","model":"phi-4-14b","place":252,"score":56.1,"version":"Phi 4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":253,"score":55.2,"version":"Qwen3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":254,"score":55.1,"version":"K2 V2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":255,"score":55.1,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":256,"score":54.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":257,"score":54.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":258,"score":54.71,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":259,"score":54.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":260,"score":54.55,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-7b-rl","place":261,"score":54.4,"version":"MiMo 7B"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":262,"score":54.3,"version":"EXAONE Deep"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":263,"score":54.1,"version":"Llama 3.1"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":264,"score":54,"version":"Llama 3.1"},{"company":"aquif","family":"Aquif","model":"aquif-3.6-1b","place":265,"score":52.8,"version":"Aquif 3.6"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":266,"score":52.53,"version":"Kanana 2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":267,"score":52,"version":"EXAONE 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":268,"score":52,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":269,"score":51,"version":"Gemini 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":270,"score":51,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":271,"score":50.6,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":272,"score":50.51,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":273,"score":50.5,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":274,"score":50.5,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":275,"score":50.3,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":276,"score":49.8,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":277,"score":49.66,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":278,"score":49.49,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":279,"score":49,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":280,"score":49,"version":"Qwen2.5"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":281,"score":48.5,"version":"K2 V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":282,"score":48.1,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":283,"score":48,"version":"GPT-4"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":284,"score":48,"version":"Youtu-LLM"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":285,"score":47.64,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":286,"score":47.47,"version":"Mistral 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":287,"score":47.47,"version":"Qwen3"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":288,"score":46.97,"version":"HY"},{"company":"weibo","family":"VibeThinker","model":"vibethinker-1.5b","place":289,"score":46.7,"version":"VibeThinker"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":290,"score":46.63,"version":"Llama 3.3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":291,"score":45.5,"version":"DeepSeek R1"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":292,"score":45.25,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":293,"score":44.95,"version":"Qwen2.5"},{"company":"tencent","family":"WeDLM","model":"wedlm-8b-instruct","place":294,"score":44.95,"version":"WeDLM"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":295,"score":43.94,"version":"Llama 4"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":296,"score":43.94,"version":"Trinity Large"},{"company":"mistral","family":"Mistral","model":"mistral-large-3-base","place":297,"score":43.9,"version":"Mistral 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":298,"score":43.8,"version":"SmollM3"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":299,"score":43.43,"version":"Seed 1.6"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":300,"score":43.1,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":301,"score":42.93,"version":"Kanana 2"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":302,"score":42.92,"version":"Motif 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":303,"score":42.42,"version":"Kanana 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":304,"score":42.4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":305,"score":42.4,"version":"Gemma 3"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-nano-1b","place":306,"score":42,"version":"Aquif 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":307,"score":41.92,"version":"DeepSeek V3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":308,"score":41.7,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-instruct","place":309,"score":41.41,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":310,"score":41.06,"version":"Llama 3.1"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":311,"score":40.91,"version":"Trinity Large"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":312,"score":40.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":313,"score":40.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":314,"score":40.9,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":315,"score":40.74,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":316,"score":40.57,"version":"Falcon H1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":317,"score":40.4,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":318,"score":40.1,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":319,"score":39.9,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":320,"score":39.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":321,"score":39.3,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":322,"score":38.89,"version":"LFM2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":323,"score":38.72,"version":"Falcon H1"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":324,"score":37.86,"version":"LFM2.5"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":325,"score":37.21,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":326,"score":36.93,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":327,"score":36.9,"version":"Qwen3"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":328,"score":36.38,"version":"Solar Pro"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":329,"score":36.38,"version":"Gemma 2"},{"company":"aquif","family":"Aquif","model":"aquif-3-3.2b","place":330,"score":36.1,"version":"Aquif 3"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-2.7b","place":331,"score":35.8,"version":"Aquif 3.5"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":332,"score":35.78,"version":"Phi 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":333,"score":35.7,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":334,"score":35.26,"version":"Llama 3.1"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":335,"score":35.2,"version":"Moonlight"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":336,"score":35.19,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":337,"score":34.9,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":338,"score":34.85,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":339,"score":34.68,"version":"Falcon 3"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":340,"score":34.38,"version":"Phi 3.5 MoE"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":341,"score":34.3,"version":"Gemma 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":342,"score":33.6,"version":"DeepSeek R1"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":343,"score":33.5,"version":"InternVL"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":344,"score":33.5,"version":"Mi:dm 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":345,"score":33.33,"version":"Gemma 3"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":346,"score":33.1,"version":"EXAONE 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":347,"score":33,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":348,"score":32.66,"version":"OLMo 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":349,"score":32.2,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":350,"score":31.8,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":351,"score":31.31,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":352,"score":31.31,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":353,"score":30.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":354,"score":30.8,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":355,"score":30.3,"version":"Qwen2.5 VL"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":356,"score":30.3,"version":"OLMo 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":357,"score":29.97,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":358,"score":29.4,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":359,"score":29.29,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":360,"score":28.8,"version":"Gemma 2"},{"company":"servicenow","family":"Apriel","model":"apriel-5b-instruct","place":361,"score":28.4,"version":"Apriel"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b","place":362,"score":28.4,"version":"EXAONE 3.5"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b-instruct","place":363,"score":28.1,"version":"EXAONE 3.5"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":364,"score":28,"version":"MobileLLM R1"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":365,"score":27.95,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":366,"score":27.9,"version":"Qwen3"},{"company":"arcee","family":"AFM","model":"afm-4.5b","place":367,"score":27.8,"version":"AFM"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":368,"score":27.78,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-6b","place":369,"score":27.7,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-350m-math","place":370,"score":27.4,"version":"LFM2"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":371,"score":27.3,"version":"Llama 3.1"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":372,"score":26.77,"version":"EuroLLM"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-mini-instruct","place":373,"score":26.6,"version":"Mi:dm 2.0"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":374,"score":26.3,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":375,"score":25.59,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":376,"score":25.4,"version":"Gemma 3"},{"company":"microsoft","family":"Phi","model":"phi-4-mini-3.8b","place":377,"score":25.2,"version":"Phi 4 mini"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":378,"score":25.08,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":379,"score":25.08,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":380,"score":24.7,"version":"Gemma 2"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":381,"score":24.7,"version":"Moonlight"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":382,"score":24.34,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":383,"score":24.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":384,"score":24.24,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":385,"score":24.24,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":386,"score":24.24,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":387,"score":21.6,"version":"Llama 3.1"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":388,"score":21.38,"version":"Apertus"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":389,"score":21.2,"version":"Llama 3.0"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":390,"score":19.7,"version":"SmollM3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":391,"score":19.2,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":392,"score":19.2,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":393,"score":16.57,"version":"Llama 3.2"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":94.3,"topic":"Graduate-level QA","total_models":1613,"url":"https://gpqa.github.io/"},{"bench_models":10,"description":"Graph traversal/GraphWalks benchmark (BFS variant) for long-context reasoning.","github":"","has_recent":0,"id":"graphwalks-bfs","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GraphWalks BFS","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":98,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":92,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":3,"score":85.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":84.2,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":5,"score":82.5,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":80.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":79.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":8,"score":72,"version":"GPT-4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":9,"score":68.9,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":10,"score":64.1,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":98,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"Graph traversal/GraphWalks benchmark (Parents variant) for long-context reasoning.","github":"","has_recent":0,"id":"graphwalks-parents","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GraphWalks Parents","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":1,"score":100,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":99.7,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":99.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":99.7,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":99,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":6,"score":97.6,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":7,"score":96.6,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":96.2,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":93,"version":"Seed 2.0"}],"stars":"","top_company":"bytedance","top_model":"Seed2.0 Lite","top_model_family":"Seed","top_model_id":"seed2.0-lite","top_model_version":"Seed 2.0","top_open_source":0,"top_percent":100,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":6,"description":"GRE quantitative section evaluated via majority voting over 16 samples.","github":"","has_recent":0,"id":"gre-math-maj16","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GRE Math maj@16","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2-7b","place":1,"score":58.5,"version":"Qwen2"},{"company":"mistral","family":"Mathstral","model":"mathstral-7b","place":2,"score":56.9,"version":"Mathstral"},{"company":"google","family":"Gemma 2","model":"gemma2-9b","place":3,"score":52.3,"version":"Gemma 2"},{"company":"zai","family":"GLM","model":"glm4-9b","place":4,"score":46.2,"version":"GLM 4"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":5,"score":44.6,"version":"DeepSeek Math"},{"company":"meta","family":"Llama","model":"llama3-8b","place":6,"score":26.2,"version":"Llama 3"}],"stars":"","top_company":"qwen","top_model":"Qwen2 7B","top_model_family":"Qwen","top_model_id":"qwen2-7b","top_model_version":"Qwen2","top_open_source":1,"top_percent":58.5,"topic":"Math (standardized tests)","total_models":1613,"url":""},{"bench_models":12,"description":"Accuracy on the Ground-UI-1K grounding benchmark.","github":"","has_recent":0,"id":"ground-ui-1k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ground-UI-1K","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":1,"score":85.4,"version":"Qwen2.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":2,"score":84.5,"version":"Holo1.5"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":3,"score":84.2,"version":"UI-TARS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":4,"score":84,"version":"Claude 4"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":5,"score":84,"version":"Holo1.5"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":6,"score":83.2,"version":"Holo1.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":7,"score":82.3,"version":"UI-Venus"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":8,"score":80.7,"version":"Qwen2.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":9,"score":78.5,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":10,"score":76.4,"version":"Qwen2.5 VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":11,"score":75.5,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":12,"score":74.75,"version":"Holo1"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5-VL 72B","top_model_family":"Qwen","top_model_id":"qwen2.5-vl-72b","top_model_version":"Qwen2.5 VL","top_open_source":1,"top_percent":85.4,"topic":"GUI grounding","total_models":1613,"url":""},{"bench_models":4,"description":"GSM-Infinite Hard benchmark at 128K context.","github":"","has_recent":0,"id":"gsm-infinite-hard-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM-Infinite Hard (128K)","paper":"","placements":[{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":1,"score":29,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":2,"score":28.7,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":3,"score":25.7,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":4,"score":8.8,"version":"Kimi K2"}],"stars":"","top_company":"xiaomi","top_model":"MiMo V2 Flash Base","top_model_family":"MiMo","top_model_id":"mimo-v2-flash-base","top_model_version":"MiMo V2","top_open_source":1,"top_percent":29,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"GSM-Infinite Hard benchmark at 16K context.","github":"","has_recent":0,"id":"gsm-infinite-hard-16k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM-Infinite Hard (16K)","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":1,"score":50.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":2,"score":41.5,"version":"DeepSeek V3.1"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":3,"score":37.7,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":4,"score":34.6,"version":"Kimi K2"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.2-Exp","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.2-exp","top_model_version":"DeepSeek V3.2","top_open_source":1,"top_percent":50.4,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"GSM-Infinite Hard benchmark at 32K context.","github":"","has_recent":0,"id":"gsm-infinite-hard-32k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM-Infinite Hard (32K)","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":1,"score":45.2,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":2,"score":38.8,"version":"DeepSeek V3.1"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":3,"score":33.7,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":4,"score":26.1,"version":"Kimi K2"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.2-Exp","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.2-exp","top_model_version":"DeepSeek V3.2","top_open_source":1,"top_percent":45.2,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"GSM-Infinite Hard benchmark at 64K context.","github":"","has_recent":0,"id":"gsm-infinite-hard-64k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM-Infinite Hard (64K)","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":1,"score":34.7,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":2,"score":32.6,"version":"DeepSeek V3.2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":3,"score":31.5,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":4,"score":16,"version":"Kimi K2"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.1","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.1","top_model_version":"DeepSeek V3.1","top_open_source":1,"top_percent":34.7,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":21,"description":"Enhanced GSM-style grade-school math benchmark variant.","github":"","has_recent":0,"id":"gsm-plus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM-Plus","paper":"","placements":[{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":1,"score":89.74,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":2,"score":89.71,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":3,"score":89.69,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":89.41,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":5,"score":89.23,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":6,"score":87.18,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":7,"score":86.55,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":8,"score":86.5,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":9,"score":85.88,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":10,"score":85.56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":11,"score":82.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":12,"score":74.1,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":13,"score":72.8,"version":"SmollM3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":14,"score":68.38,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":15,"score":68.3,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":16,"score":64.76,"version":"LFM2"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":17,"score":60.75,"version":"LFM2"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":18,"score":59.2,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":19,"score":59.14,"version":"Granite 4.0"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":20,"score":58.91,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":21,"score":56.16,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":22,"score":38.68,"version":"Llama 3.2"}],"stars":"","top_company":"antgroup","top_model":"LLaDA2.0 Flash","top_model_family":"LLaDA","top_model_id":"llada2.0-flash","top_model_version":"LLaDA 2.0","top_open_source":1,"top_percent":89.74,"topic":"Math (grade-school, enhanced)","total_models":1613,"url":""},{"bench_models":4,"description":"Symbolic reasoning variant of GSM that tests algebraic manipulation and arithmetic with structured problems.","github":"https://github.com/kosukearai/gsm-symbolic","has_recent":0,"id":"gsm-symbolic","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM-Symbolic","paper":"","placements":[{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":1,"score":87.38,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":2,"score":82.27,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":3,"score":81.45,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":4,"score":79.82,"version":"Granite 4.0"}],"stars":"","top_company":"IBM","top_model":"Granite-4.0-H-Small","top_model_family":"Granite","top_model_id":"granite-4.0-h-small","top_model_version":"Granite 4.0","top_open_source":1,"top_percent":87.38,"topic":"Math reasoning","total_models":1613,"url":"https://github.com/kosukearai/gsm-symbolic"},{"bench_models":254,"description":"Grade-school math word problems requiring multi-step reasoning.","github":"https://github.com/openai/grade-school-math","has_recent":0,"id":"gsm8k","leaderboard":"https://llm-stats.com/benchmarks/gsm8k","lower_is_better":false,"metric":"percent","name":"GSM8K","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":1,"score":97.3,"version":"Kimi K2"},{"company":"openai","family":"o1","model":"o1","place":2,"score":97.1,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":3,"score":97,"version":"GPT-4.5"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":4,"score":96.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":5,"score":96.8,"version":"Llama 3.1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":6,"score":96.6,"version":"ERNIE 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-3.5","place":7,"score":96.4,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":8,"score":96.4,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":9,"score":96.36,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":10,"score":96.36,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":11,"score":96.3,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":12,"score":96.26,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":13,"score":96.13,"version":"Motif 2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":14,"score":96.1,"version":"OLMo 3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":15,"score":96.1,"version":"GLM 4.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":16,"score":96.06,"version":"LLaDA 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":17,"score":95.98,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":18,"score":95.9,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":19,"score":95.9,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":20,"score":95.9,"version":"Qwen2.5"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":21,"score":95.83,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":22,"score":95.8,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":23,"score":95.7,"version":"OLMo 3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":24,"score":95.53,"version":"Mistral 3.2"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":25,"score":95.53,"version":"Motif 2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":26,"score":95.5,"version":"Tulu 3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":27,"score":95.45,"version":"Ling 2.0"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":28,"score":95.4,"version":"MiniMax M2"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":29,"score":95.4,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":30,"score":95.22,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":31,"score":95.1,"version":"DeepSeek V2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":32,"score":95.1,"version":"Llama 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-3","place":33,"score":95,"version":"Claude 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":34,"score":94.92,"version":"Motif 2"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":35,"score":94.8,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":36,"score":94.5,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":37,"score":94.4,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":38,"score":94.4,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":39,"score":94.39,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":40,"score":94.2,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":41,"score":94.1,"version":"DeepSeek V3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":42,"score":93.85,"version":"Motif 2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":43,"score":93.71,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":44,"score":93.7,"version":"Llama 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":45,"score":93.6,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":46,"score":93.56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":47,"score":93.5,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":48,"score":93.5,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":49,"score":93.5,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":50,"score":93.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":51,"score":93.33,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":52,"score":93.1,"version":"Seed 1.6"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":53,"score":93.1,"version":"Kanana 2"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":54,"score":93,"version":"Llama 3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":55,"score":92.7,"version":"Hermes 3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":56,"score":92.5,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":57,"score":92.49,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":58,"score":92.47,"version":"OLMo 2"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":59,"score":92.4,"version":"K2 V2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":60,"score":92.34,"version":"Nemotron 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":61,"score":92.3,"version":"MiMo V2"},{"company":"tencent","family":"WeDLM","model":"wedlm-8b-instruct","place":62,"score":92.27,"version":"WeDLM"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":63,"score":92.12,"version":"Llama 3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":64,"score":92.1,"version":"Kimi K2"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":65,"score":92,"version":"K2 V2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":66,"score":91.89,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":67,"score":91.81,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":68,"score":91.7,"version":"GPT-4o"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":69,"score":91.6,"version":"Mi:dm 2.0"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":70,"score":91.58,"version":"Kanana 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":71,"score":91.4,"version":"DeepSeek V3.1"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":72,"score":91.36,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":73,"score":91.2,"version":"Qwen2.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":74,"score":91.1,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":75,"score":91.1,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":76,"score":90.4,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":77,"score":90.37,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":78,"score":90.37,"version":"Qwen3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":79,"score":90,"version":"Hermes 3"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":80,"score":89.99,"version":"Seed 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":81,"score":89.92,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-instruct","place":82,"score":89.91,"version":"Qwen3"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":83,"score":89.69,"version":"Solar Pro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":84,"score":89.5,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":85,"score":89.3,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":86,"score":89.01,"version":"LLaDA 2.0"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":87,"score":89,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":88,"score":88.78,"version":"Qwen3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":89,"score":88.78,"version":"JoyAI-LLM"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":90,"score":88.4,"version":"Mixtral"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":91,"score":88.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":92,"score":88,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":93,"score":87.72,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":94,"score":87.7,"version":"Llama 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":95,"score":87.6,"version":"DeepSeek V3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":96,"score":87.6,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":97,"score":87.57,"version":"DeepSeek V3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":98,"score":87.49,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":99,"score":87.27,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":100,"score":86.92,"version":"Granite 4.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":101,"score":86.3,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":102,"score":85.8,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0-instruct","place":103,"score":85.6,"version":"Bielik v3.0"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":104,"score":85.6,"version":"LFM2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":105,"score":85.6,"version":"Qwen3"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":106,"score":85.54,"version":"EuroLLM"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.5-instruct","place":107,"score":85.52,"version":"Bielik v2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":108,"score":85.45,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":109,"score":85.2,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":110,"score":85.14,"version":"Kanana 1.5"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":111,"score":85.06,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":112,"score":84.76,"version":"Falcon H1"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":113,"score":84.76,"version":"Phi 3"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":114,"score":84.7,"version":"Llama 3.1 Nemotron"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":115,"score":84.69,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":116,"score":84.5,"version":"Llama 3.1"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":117,"score":84.38,"version":"LFM2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":118,"score":84.3,"version":"Tulu 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":119,"score":84.1,"version":"Nemotron Nano v2"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":120,"score":84.08,"version":"HY"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":121,"score":83.8,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.6-instruct","place":122,"score":83.78,"version":"Bielik v2.6"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":123,"score":83.62,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":124,"score":83.4,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":125,"score":83.2,"version":"Llama 3.0"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-mini-instruct","place":126,"score":83.1,"version":"Mi:dm 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":127,"score":83.09,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":128,"score":82.71,"version":"Kanana 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":129,"score":82.64,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":130,"score":82.6,"version":"Gemma 3"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b-instruct","place":131,"score":82.5,"version":"EXAONE 3.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":132,"score":82.49,"version":"Llama 3.1"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":133,"score":82.41,"version":"LFM2"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":134,"score":82.34,"version":"Falcon H1"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":135,"score":82.26,"version":"Phi 3.5 MoE"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":136,"score":82,"version":"Gemma 3"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":137,"score":81.88,"version":"GLM 4.7"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":138,"score":81.65,"version":"Falcon H1"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":139,"score":81.65,"version":"Granite 3.2"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":140,"score":81.58,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":141,"score":81.5,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.3-instruct","place":142,"score":81.27,"version":"Bielik v2.3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":143,"score":81.2,"version":"Llama 3.1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":144,"score":81.12,"version":"SmollM3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.2-instruct","place":145,"score":81.12,"version":"Bielik v2.2"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":146,"score":81.1,"version":"Llama 3.1"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":147,"score":81.1,"version":"EXAONE 3.5"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":148,"score":81.05,"version":"Falcon 3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":149,"score":80.89,"version":"Granite 3.3"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":150,"score":80.6,"version":"Llama 3"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":151,"score":80.6,"version":"DeepSeek Math"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.1-instruct","place":152,"score":80.59,"version":"Bielik v2.1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":153,"score":80.5,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":154,"score":80.44,"version":"Qwen3"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":155,"score":80.36,"version":"Trinity Large"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":156,"score":80.04,"version":"Apertus"},{"company":"radicalnumerics","family":"RND1","model":"rnd1-base-0910","place":157,"score":80,"version":"RND1"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":158,"score":80,"version":"Ministral"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":159,"score":79.83,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":160,"score":79.7,"version":"Gemma 2"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":161,"score":79.4,"version":"GLM 4.5"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":162,"score":79.38,"version":"Phi 3"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":163,"score":79.15,"version":"Granite 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":164,"score":78.92,"version":"Qwen3"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":165,"score":78.92,"version":"Ouro"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":166,"score":78.7,"version":"HY"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":167,"score":78.6,"version":"Mixtral"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":168,"score":78.47,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":169,"score":78.17,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":170,"score":77.41,"version":"Gemma 3"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":171,"score":77.33,"version":"HY"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":172,"score":77.26,"version":"Llama 3.2"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":173,"score":77.2,"version":"Dream 7B"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":174,"score":77.2,"version":"Moonlight"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":175,"score":77.18,"version":"Gemma 3"},{"company":"mistral","family":"Mathstral","model":"mathstral-7b","place":176,"score":77.1,"version":"Mathstral"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":177,"score":76.57,"version":"Kanana 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":178,"score":76.4,"version":"OLMo 2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.0-instruct","place":179,"score":76.27,"version":"Bielik v2.0"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":180,"score":76.2,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":181,"score":75.97,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama3-8b","place":182,"score":75.4,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":183,"score":75.21,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":184,"score":74.98,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":185,"score":74.68,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":186,"score":74.6,"version":"Qwen2.5"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":187,"score":74.4,"version":"Mixtral"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0","place":188,"score":74.15,"version":"Bielik v3.0"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3-instruct","place":189,"score":73.69,"version":"Bielik v3"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":190,"score":73.42,"version":"Granite 4.0"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b","place":191,"score":73.1,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":192,"score":72.86,"version":"Qwen3"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5-0106-gemma","place":193,"score":72.86,"version":"OpenChat 3.5 0106"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":194,"score":72.48,"version":"Granite 3.3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":195,"score":72.18,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":196,"score":71.95,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":197,"score":71,"version":"Gemma 3"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":198,"score":70.9,"version":"LLaDA"},{"company":"cohere","family":"Command","model":"command-r-plus-08-2024","place":199,"score":70.7,"version":"Command"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":200,"score":70.28,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":201,"score":70.05,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":202,"score":69.6,"version":"Llama 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":203,"score":69.6,"version":"Granite 4.0"},{"company":"google","family":"Gemma 2","model":"gemma2-9b","place":204,"score":69.5,"version":"Gemma 2"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":205,"score":69.1,"version":"Marin 32B"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":206,"score":68.69,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"meta-llama-3-8b-instruct","place":207,"score":68.69,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":208,"score":68.46,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":209,"score":68.39,"version":"Falcon H1"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":210,"score":68,"version":"K2 V2"},{"company":"allenai","family":"Bolmo","model":"bolmo-7b","place":211,"score":68,"version":"Bolmo 7B"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2","place":212,"score":67.78,"version":"Bielik v2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b","place":213,"score":67.63,"version":"Qwen1.5"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":214,"score":67.55,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":215,"score":67.2,"version":"Llama 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":216,"score":67.02,"version":"Granite 3.2"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":217,"score":64.75,"version":"Solar Mini"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":218,"score":64.52,"version":"LFM2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3","place":219,"score":63.61,"version":"Bielik v3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":220,"score":62.85,"version":"Gemma 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m-base","place":221,"score":61.6,"version":"MobileLLM R1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":222,"score":61.3,"version":"Qwen2.5"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":223,"score":61.2,"version":"Magpie"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b-base","place":224,"score":60.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":225,"score":60.73,"version":"Qwen2.5"},{"company":"alephalpha","family":"TFree","model":"tfree-hat-pretrained-7b-base","place":226,"score":60.7,"version":"TFree Hat"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":227,"score":59.59,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen-14b","place":228,"score":58.98,"version":"Qwen"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":229,"score":58.4,"version":"Mixtral"},{"company":"liquidai","family":"LFM","model":"lfm2-1.2b","place":230,"score":58.3,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":231,"score":57.54,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":232,"score":57.47,"version":"Qwen2.5"},{"company":"cohere","family":"Command","model":"command-r-08-2024","place":233,"score":56.6,"version":"Command"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":234,"score":55.04,"version":"HY"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":235,"score":54.7,"version":"Marin 32B"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":236,"score":53.6,"version":"Llama 2"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":237,"score":52.77,"version":"gemma-7b"},{"company":"aquif","family":"Aquif","model":"aquif-3-0.4b","place":238,"score":52.1,"version":"Aquif 3"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":239,"score":52.1,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":240,"score":51.4,"version":"Qwen3"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-a0.6b","place":241,"score":50.7,"version":"Aquif 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":242,"score":50,"version":"Mistral"},{"company":"zai","family":"GLM","model":"glm4-9b","place":243,"score":48.8,"version":"GLM 4"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":244,"score":48.2,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":245,"score":46.4,"version":"Gemma 2"},{"company":"liquidai","family":"LFM","model":"lfm2-700m","place":246,"score":46.4,"version":"LFM2"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":247,"score":45.19,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":248,"score":44.1,"version":"Llama 1"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":249,"score":44.05,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":250,"score":42.8,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":251,"score":42.15,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":252,"score":40.03,"version":"Mistral v0.2"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":253,"score":40,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-2-0425-1b-base","place":254,"score":39.8,"version":"OLMo 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-360m-base","place":255,"score":39.4,"version":"MobileLLM R1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":256,"score":39.04,"version":"Llama 3.2"},{"company":"pleias","family":"Baguettotron","model":"baguettotron-321m","place":257,"score":39,"version":"Baguettotron"},{"company":"IBM","family":"Granite","model":"granite-4-h-300m","place":258,"score":39,"version":"Granite 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":259,"score":38.51,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.1","place":260,"score":37.83,"version":"Mistral v0.1"},{"company":"hku","family":"EvaByte","model":"evabyte-6.5b","place":261,"score":36.7,"version":"EvaByte 6.5B"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":262,"score":36.5,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":263,"score":36.5,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":264,"score":36.47,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":265,"score":35.71,"version":"Llama 3.2"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.2","place":266,"score":34.72,"version":"Mistral v0.2"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":267,"score":34.3,"version":"Llama 2"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":268,"score":34,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":269,"score":33.66,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2-7b","place":270,"score":32.7,"version":"Qwen2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":271,"score":31.8,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":272,"score":31,"version":"SmolLM2"},{"company":"IBM","family":"Granite","model":"granite-4-300m","place":273,"score":31,"version":"Granite 4"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b-chat","place":274,"score":30.63,"version":"Qwen1.5"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":275,"score":30.1,"version":"LFM2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-instruct-v0.1","place":276,"score":29.95,"version":"Bielik v0.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-v0.1","place":277,"score":29.49,"version":"Bielik v0.1"},{"company":"meta","family":"Llama","model":"llama-1b","place":278,"score":26.8,"version":"Llama 1"},{"company":"meta","family":"BLT","model":"blt-7b","place":279,"score":24.2,"version":"BLT 7B"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":280,"score":20.8,"version":"CodeLlama-7B"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":281,"score":17.7,"version":"Gemma 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-140m-base","place":282,"score":16.3,"version":"MobileLLM R1"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":283,"score":16,"version":"Llama 2"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.1","place":284,"score":14.25,"version":"Mistral v0.1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-7b-chat","place":285,"score":13.57,"version":"Qwen1.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-instruct","place":286,"score":11,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":287,"score":7.2,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":288,"score":7.05,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":289,"score":7.05,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":290,"score":6.8,"version":"Llama 3.2"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":291,"score":5.5,"version":"SmolLM1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":292,"score":5,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":293,"score":4.62,"version":"SmolLM1"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":294,"score":3,"version":"Gemma 3"},{"company":"loka","family":"TRLM","model":"trlm-135m","place":295,"score":2.59,"version":"TRLM"},{"company":"google","family":"Gemma","model":"gemma-3-pt-1b","place":296,"score":2.4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":297,"score":2.05,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":298,"score":1.6,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":299,"score":1.4,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":300,"score":1.4,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":301,"score":1.1,"version":"Gemma 3"}],"stars":"1322","top_company":"moonshotai","top_model":"Kimi K2 Instruct","top_model_family":"Kimi","top_model_id":"kimi-k2-instruct","top_model_version":"Kimi K2","top_open_source":1,"top_percent":97.3,"topic":"Math (grade-school)","total_models":1613,"url":"https://huggingface.co/datasets/openai/gsm8k"},{"bench_models":2,"description":"German translation of the GSM8K grade-school math word problems.","github":"","has_recent":0,"id":"gsm8k-de","leaderboard":"","lower_is_better":false,"metric":"comp_acc","name":"GSM8K (DE)","paper":"","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":0.63,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":2,"score":0.139,"version":"Llama 3.3"}],"stars":"","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":0.63,"topic":"Math (grade-school, German)","total_models":1613,"url":""},{"bench_models":4,"description":"Korean translation of the GSM8K grade-school math word problems.","github":"","has_recent":0,"id":"gsm8k-ko","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM8K-Ko","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":1,"score":88.1,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":2,"score":81.43,"version":"Kanana 1.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":3,"score":77.48,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":4,"score":70.43,"version":"Kanana 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b","top_model_version":"Qwen3","top_open_source":1,"top_percent":88.1,"topic":"Math (grade-school, Korean)","total_models":1613,"url":""},{"bench_models":3,"description":"Harder subset/setting of GSM8K grade-school math problems.","github":"","has_recent":0,"id":"gsm8k-platinum","leaderboard":"","lower_is_better":false,"metric":"percent","name":"GSM8K Platinum","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":1,"score":89.6,"version":"Kimi Linear"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":2,"score":89.5,"version":"OLMo 2"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":3,"score":79.4,"version":"Moonlight"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-Linear-Base","top_model_family":"Kimi","top_model_id":"kimi-linear-base","top_model_version":"Kimi Linear","top_open_source":1,"top_percent":89.6,"topic":"Math (grade-school, hard)","total_models":1613,"url":"https://github.com/openai/grade-school-math"},{"bench_models":11,"description":"LiveCodeBench GSO benchmark.","github":"","has_recent":0,"id":"gso-benchmark","leaderboard":"https://gso-bench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"GSO Benchmark","paper":"","placements":[{"company":"openai","family":"o3","model":"o3-high","place":1,"score":8.8,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":2,"score":6.9,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":6.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":4,"score":4.9,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":5,"score":4.9,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":6,"score":4.9,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":7,"score":4.6,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":8,"score":3.8,"version":"Claude 3.7"},{"company":"openai","family":"o4","model":"o4-mini-high","place":9,"score":3.6,"version":"o4"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":10,"score":2.9,"version":"GLM 4.5"},{"company":"openai","family":"o3","model":"o3-mini-high","place":11,"score":1.3,"version":"o3"}],"stars":"","top_company":"openai","top_model":"o3-high","top_model_family":"o3","top_model_id":"o3-high","top_model_version":"o3","top_open_source":0,"top_percent":8.8,"topic":"Code generation","total_models":1613,"url":"https://livecodebench.github.io/gso.html"},{"bench_models":8,"description":"Korean language understanding benchmark evaluating knowledge and reasoning.","github":"","has_recent":0,"id":"hae-rae-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HAE-RAE Bench","paper":"","placements":[{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":1,"score":90.65,"version":"Kanana 1.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":2,"score":88.91,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":3,"score":88.73,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":4,"score":75.57,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":5,"score":75.57,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":6,"score":72.04,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":7,"score":65.34,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":8,"score":57.39,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":9,"score":53.41,"version":"Qwen3"}],"stars":"","top_company":"kakao","top_model":"Kanana-1.5-32.5B-Base","top_model_family":"Kanana","top_model_id":"kanana-1.5-32.5b-base","top_model_version":"Kanana 1.5","top_open_source":1,"top_percent":90.65,"topic":"Korean language understanding","total_models":1613,"url":"https://huggingface.co/datasets/HAERAE-HUB/HAE_RAE_BENCH"},{"bench_models":39,"description":"Benchmark for evaluating hallucination tendencies in multimodal LLMs.","github":"https://github.com/FuxiaoLiu/HallusionBench","has_recent":0,"id":"hallusionbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HallusionBench","paper":"https://arxiv.org/abs/2310.14566","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":71.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":2,"score":70,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":69.9,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":69.8,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":68.6,"version":"Gemini 3"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":6,"score":68,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":7,"score":67.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":8,"score":67.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":9,"score":66.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":10,"score":66.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":66,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":12,"score":65.7,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":13,"score":65.4,"version":"GLM 4.5V"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":14,"score":65.2,"version":"GPT-5.2"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":15,"score":64.83,"version":"dots.vlm1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":16,"score":64.8,"version":"Claude 4.5"},{"company":"stepfun","family":"Step","model":"step-3","place":17,"score":64.2,"version":"Step 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":64.1,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":19,"score":64.1,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":20,"score":63.9,"version":"Seed 1.8"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":21,"score":63.8,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":22,"score":63.7,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":23,"score":63.49,"version":"Seed 1.5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":24,"score":63.2,"version":"GLM 4.1V"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":25,"score":63.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":26,"score":63.07,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":27,"score":62.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":28,"score":61.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":29,"score":61.1,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":30,"score":61.1,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":31,"score":60.4,"version":"Claude 4.1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":32,"score":60.4,"version":"ERNIE 4.5 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":33,"score":60.3,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":34,"score":59.9,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":35,"score":59.8,"version":"Kimi VL"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":36,"score":59.1,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":37,"score":58.1,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":38,"score":57.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":39,"score":56.5,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":40,"score":55.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":41,"score":54.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":42,"score":54.5,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":43,"score":53.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":44,"score":51.4,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":45,"score":45.8,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":46,"score":39.3,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":71.4,"topic":"Multimodal hallucination","total_models":1613,"url":"https://hallusionbench.github.io/"},{"bench_models":5,"description":"Harmfulness and safety compliance benchmark across a variety of risky prompts.","github":"","has_recent":0,"id":"harmbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HarmBench","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":1,"score":94.8,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":2,"score":94.2,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":3,"score":57.3,"version":"IQuest Coder V1"},{"company":"mbzuai","family":"K2","model":"k2-think","place":4,"score":56,"version":"K2-THINK"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":5,"score":47.5,"version":"Qwen2.5"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Thinking","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-thinking","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":94.8,"topic":"Safety","total_models":1613,"url":""},{"bench_models":3,"description":"Harmful question set testing models' ability to avoid unsafe answers.","github":"https://github.com/declare-lab/red-instruct","has_recent":0,"id":"harmfulq","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HarmfulQA","paper":"https://arxiv.org/abs/2308.09662","placements":[{"company":"mbzuai","family":"K2","model":"k2-think","place":1,"score":99,"version":"K2-THINK"},{"company":"openai","family":"GPT","model":"gpt-4","place":2,"score":79.4,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"chatgpt","place":3,"score":74.3,"version":"ChatGPT"}],"stars":"104","top_company":"mbzuai","top_model":"K2-THINK","top_model_family":"K2","top_model_id":"k2-think","top_model_version":"K2-THINK","top_open_source":1,"top_percent":99,"topic":"Safety","total_models":1613,"url":"https://huggingface.co/datasets/d-llm/HarmfulQA"},{"bench_models":31,"description":"Comprehensive medical knowledge and clinical reasoning benchmark across specialties and tasks.","github":"","has_recent":0,"id":"healthbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HealthBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":67.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking-high","place":2,"score":67.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":3,"score":63.3,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":4,"score":62.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":5,"score":59.82,"version":"Qwen3 Next"},{"company":"openai","family":"o3","model":"o3","place":6,"score":59.8,"version":"o3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":7,"score":58,"version":"Kimi K2"},{"company":"antgroup","family":"Ring","model":"ring-1t","place":8,"score":57.93,"version":"Ring"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":9,"score":57.7,"version":"Seed 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":10,"score":55.56,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.2-instant","place":11,"score":55.4,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":12,"score":54.99,"version":"Seed OSS"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":13,"score":54.44,"version":"Ring"},{"company":"openai","family":"GPT","model":"gpt-5.3-instant","place":14,"score":54.1,"version":"GPT-5.3"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":15,"score":51.6,"version":"Ring 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":16,"score":51.6,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":17,"score":51.2,"version":"Seed 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":18,"score":50.19,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":19,"score":49.57,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":20,"score":49.39,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":21,"score":49.1,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":22,"score":47.61,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":23,"score":46.9,"version":"DeepSeek V3.2"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":24,"score":46.61,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":25,"score":44.2,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":26,"score":43.8,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":27,"score":37.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":28,"score":36.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":29,"score":32,"version":"GPT-4o"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":30,"score":30,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":31,"score":28.7,"version":"Claude 4.5"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":67.2,"topic":"Medical QA","total_models":1613,"url":""},{"bench_models":13,"description":"Challenging subset of HealthBench focusing on complex, ambiguous clinical cases.","github":"","has_recent":0,"id":"healthbench-hard","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HealthBench-Hard","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":46.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":42,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":3,"score":38.6,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":4,"score":31.6,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":5,"score":29.1,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-instant","place":6,"score":26.8,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.3-instant","place":7,"score":25.9,"version":"GPT-5.3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":21.5,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":9,"score":20,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":10,"score":15.3,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":11,"score":15,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":12,"score":11,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":10.9,"version":"Claude 4.5"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":46.2,"topic":"Medical QA (hard)","total_models":1613,"url":""},{"bench_models":3,"description":"Measures hallucination and unsafe medical advice under hard clinical scenarios.","github":"","has_recent":0,"id":"healthbench-hard-hallucinations","leaderboard":"","lower_is_better":true,"metric":"percent","name":"HealthBench-Hard Hallucinations","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":1.6,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":12.9,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":3,"score":15.8,"version":"GPT-4o"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":1.6,"topic":"Medical hallucination safety","total_models":1613,"url":""},{"bench_models":148,"description":"Adversarial commonsense sentence completion.","github":"","has_recent":0,"id":"hellaswag","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HellaSwag","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":1,"score":96.4,"version":"DeepSeek V3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":2,"score":96.2,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":3,"score":94.6,"version":"Kimi K2"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":4,"score":91.79,"version":"JoyAI-LLM"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":5,"score":90.11,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":6,"score":89.9,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":7,"score":89.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":8,"score":89.2,"version":"DeepSeek V3.1"},{"company":"cohere","family":"Command","model":"command-r-plus-08-2024","place":9,"score":88.6,"version":"Command"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":10,"score":88.5,"version":"Mixtral"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":11,"score":88.5,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":12,"score":88.29,"version":"Qwen3"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":13,"score":88.16,"version":"Solar Mini"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":14,"score":88.13,"version":"Trinity Large"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":15,"score":88,"version":"Llama 3.0"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":16,"score":87.8,"version":"K2 V2"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":17,"score":87.6,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":18,"score":87.6,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":19,"score":87.48,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5","place":20,"score":87.3,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":21,"score":87.1,"version":"GLM 4.5"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":22,"score":87.1,"version":"Llama 2"},{"company":"cohere","family":"Command","model":"command-r-08-2024","place":23,"score":87,"version":"Command"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":24,"score":86.9,"version":"Marin 32B"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":25,"score":86.6,"version":"Mixtral"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":26,"score":86.6,"version":"Marin 32B"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":27,"score":86.51,"version":"Mistral 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":28,"score":86.42,"version":"Llama 3.1"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":29,"score":86.36,"version":"Solar Pro"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":30,"score":86.31,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":31,"score":86.3,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":32,"score":86.1,"version":"Llama 3.3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":33,"score":86.02,"version":"Gemma 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":34,"score":86,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":35,"score":86,"version":"LLaDA 2.0"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":36,"score":85.97,"version":"Phi 3.5 MoE"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":37,"score":85.9,"version":"OLMo 2"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":38,"score":85.76,"version":"Phi 3"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":39,"score":85.68,"version":"Phi 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":40,"score":85.6,"version":"Gemma 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":41,"score":85.6,"version":"LLaDA 2.1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":42,"score":85.56,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":43,"score":85.49,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":44,"score":85.4,"version":"Llama 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":45,"score":85.31,"version":"LLaDA 2.1"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":46,"score":85,"version":"Llama 3.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":47,"score":85,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":48,"score":84.97,"version":"LLaDA 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":49,"score":84.88,"version":"Mistral v0.2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":50,"score":84.8,"version":"OLMo 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":51,"score":84.4,"version":"Mixtral"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":52,"score":84.2,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":53,"score":84.2,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":54,"score":84.1,"version":"Qwen2.5"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":55,"score":84,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen-14b","place":56,"score":83.99,"version":"Qwen"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":57,"score":83.76,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":58,"score":83.68,"version":"Gemma 3"},{"company":"google","family":"PaLM","model":"palm-540b","place":59,"score":83.4,"version":"PaLM"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.1","place":60,"score":83.31,"version":"Mistral v0.1"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":61,"score":83.1,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.2","place":62,"score":83.08,"version":"Mistral v0.2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":63,"score":83,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":64,"score":82.8,"version":"Llama 1"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":65,"score":82.59,"version":"Mistral 3.2"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":66,"score":82.47,"version":"gemma-7b"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b-chat","place":67,"score":82.27,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":68,"score":82.23,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":69,"score":81.97,"version":"Llama 3.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0-instruct","place":70,"score":81.96,"version":"Bielik v3.0"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":71,"score":81.94,"version":"Falcon H1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":72,"score":81.6,"version":"Ling 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":73,"score":81.59,"version":"Ling 2.0"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":74,"score":81.4,"version":"Nemotron Nano v2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0","place":75,"score":81.38,"version":"Bielik v3.0"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":76,"score":81.3,"version":"Mistral"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":77,"score":81.2,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b","place":78,"score":81.08,"version":"Qwen1.5"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5-0106-gemma","place":79,"score":81.08,"version":"OpenChat 3.5 0106"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":80,"score":81,"version":"Mistral"},{"company":"google","family":"Chinchilla","model":"chinchilla-70b","place":81,"score":80.8,"version":"Chinchilla"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.5-instruct","place":82,"score":80.71,"version":"Bielik v2.5"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":83,"score":80.7,"version":"Llama 2"},{"company":"google","family":"PaLM","model":"palm-cont-62b","place":84,"score":80.6,"version":"PaLM"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.6-instruct","place":85,"score":80.56,"version":"Bielik v2.6"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.1-instruct","place":86,"score":80.2,"version":"Bielik v2.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.2-instruct","place":87,"score":80.16,"version":"Bielik v2.2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.3-instruct","place":88,"score":80.11,"version":"Bielik v2.3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":89,"score":80.08,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":90,"score":79.98,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2","place":91,"score":79.84,"version":"Bielik v2"},{"company":"google","family":"PaLM","model":"palm-62b","place":92,"score":79.7,"version":"PaLM"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":93,"score":79.69,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":94,"score":79.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":95,"score":79.56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":96,"score":79.51,"version":"Qwen3"},{"company":"google","family":"Gopher","model":"gopher-280b","place":97,"score":79.2,"version":"Gopher"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":98,"score":79.2,"version":"Llama 1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":99,"score":79.01,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":100,"score":78.96,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-3","place":101,"score":78.9,"version":"GPT-3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.0-instruct","place":102,"score":78.65,"version":"Bielik v2.0"},{"company":"qwen","family":"Qwen","model":"qwen1.5-7b-chat","place":103,"score":78.56,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"meta-llama-3-8b-instruct","place":104,"score":78.55,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":105,"score":77.1,"version":"Llama 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":106,"score":76.19,"version":"LLaDA 2.1"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":107,"score":76.1,"version":"Llama 1"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":108,"score":75.92,"version":"Falcon H1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":109,"score":75.71,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":110,"score":75.66,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.1","place":111,"score":75.63,"version":"Mistral v0.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":112,"score":75.58,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":113,"score":75.57,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":114,"score":74.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":115,"score":74.54,"version":"Qwen2.5"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":116,"score":74.29,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":117,"score":73.95,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3-instruct","place":118,"score":73.9,"version":"Bielik v3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":119,"score":73.54,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":120,"score":73.09,"version":"Llama 3.2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3","place":121,"score":73.01,"version":"Bielik v3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":122,"score":71.81,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":123,"score":71.4,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":124,"score":71.28,"version":"Llama 3.1"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":125,"score":70.39,"version":"EuroLLM"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":126,"score":70.24,"version":"Llama 3.3"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":127,"score":70.24,"version":"Apertus"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":128,"score":69.85,"version":"Falcon H1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":129,"score":69.02,"version":"Ling 2.0"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-instruct-v0.1","place":130,"score":68.91,"version":"Bielik v0.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":131,"score":68.89,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":132,"score":68.79,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":133,"score":68.7,"version":"SmolLM2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-v0.1","place":134,"score":67.92,"version":"Bielik v0.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":135,"score":67.73,"version":"Qwen2.5"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":136,"score":67.11,"version":"MobileLLM P1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":137,"score":67.09,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":138,"score":66.4,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":139,"score":66.1,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":140,"score":65.69,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":141,"score":65.54,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":142,"score":65.03,"version":"Llama 4"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":143,"score":64.24,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":144,"score":64.21,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":145,"score":63.74,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":146,"score":63.4,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":147,"score":63.33,"version":"Falcon H1"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":148,"score":62.9,"version":"SmolLM1"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":149,"score":62.9,"version":"CodeLlama-7B"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":150,"score":62.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":151,"score":62.23,"version":"Qwen2.5"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":152,"score":61.74,"version":"EuroLLM"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":153,"score":61.22,"version":"Apertus"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":154,"score":61.2,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":155,"score":60.9,"version":"Qwen2.5"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":156,"score":60.84,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":157,"score":60.56,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":158,"score":59.35,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":159,"score":58.53,"version":"Falcon 3"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":160,"score":58.4,"version":"MobileLLM"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":161,"score":57.25,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":162,"score":56.12,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-1b","place":163,"score":56.1,"version":"Llama 1"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":164,"score":55.63,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":165,"score":55.5,"version":"SmolLM1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":166,"score":55.31,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":167,"score":55.3,"version":"Llama 3.2"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":168,"score":54.81,"version":"OLMo 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":169,"score":52.38,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":170,"score":51.93,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":171,"score":49.27,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":172,"score":47.68,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":173,"score":44.29,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":174,"score":42.24,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":175,"score":42.17,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":176,"score":42.1,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":177,"score":42.1,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":178,"score":41.2,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":179,"score":40.9,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":180,"score":40.9,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":181,"score":37.7,"version":"Gemma 3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":182,"score":0.809,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":183,"score":0.665,"version":"Llama 3.3"}],"stars":"220","top_company":"deepseek","top_model":"DeepSeek V3 Base","top_model_family":"DeepSeek","top_model_id":"deepseek-v3-base","top_model_version":"DeepSeek V3","top_open_source":1,"top_percent":96.4,"topic":"Commonsense reasoning","total_models":1613,"url":"https://github.com/rowanz/hellaswag"},{"bench_models":2,"description":"German translation of the HellaSwag commonsense benchmark.","github":"","has_recent":0,"id":"hellaswag-de","leaderboard":"","lower_is_better":false,"metric":"norm_log_acc","name":"HellaSwag (DE)","paper":"","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":0.727,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":2,"score":0.707,"version":"Llama 3.3"}],"stars":"","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":0.727,"topic":"Commonsense reasoning (German)","total_models":1613,"url":""},{"bench_models":4,"description":"Long-context subset of the HELMET benchmark focusing on grounded question answering.","github":"","has_recent":0,"id":"helmet-longqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HELMET LongQA","paper":"","placements":[{"company":"ai21","family":"Jamba","model":"jamba-mini-1.6","place":1,"score":46.9,"version":"Jamba Mini"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":2,"score":33,"version":"Ministral"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":3,"score":29.2,"version":"Llama 3.1"},{"company":"cohere","family":"Command","model":"command-r7b","place":4,"score":9.6,"version":"Command"}],"stars":"","top_company":"ai21","top_model":"Jamba Mini 1.6","top_model_family":"Jamba","top_model_id":"jamba-mini-1.6","top_model_version":"Jamba Mini","top_open_source":0,"top_percent":46.9,"topic":"Long-context QA","total_models":1613,"url":""},{"bench_models":24,"description":"Benchmark for Long-Horizon Planning and Structured Reasoning in Virtual Worlds.","github":"","has_recent":0,"id":"herobench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HeroBench","paper":"https://arxiv.org/pdf/2508.12782","placements":[{"company":"xai","family":"Grok","model":"grok-4","place":1,"score":91.7,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":83.9,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":62.9,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":4,"score":60.6,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":5,"score":44.4,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":35,"version":"GPT-5"},{"company":"openai","family":"o4","model":"o4-mini","place":7,"score":35,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":8,"score":31.7,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":9,"score":26.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":10,"score":24.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":11,"score":21.7,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":12,"score":17.2,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":13,"score":16.1,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":14,"score":13.3,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-70b","place":15,"score":11.2,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":16,"score":10,"version":"Qwen3"},{"company":"magistral","family":"Magistral","model":"magistral-medium","place":17,"score":9.4,"version":"Magistral"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":18,"score":8.9,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":19,"score":8.3,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":20,"score":7.2,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":21,"score":3.9,"version":"Qwen3"},{"company":"gigachat","family":"GigaChat","model":"gigachat-2-max","place":22,"score":2.8,"version":"GigaChat 2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":23,"score":1.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":24,"score":0,"version":"Qwen3"}],"stars":"","top_company":"xai","top_model":"Grok 4","top_model_family":"Grok","top_model_id":"grok-4","top_model_version":"Grok 4","top_open_source":0,"top_percent":91.7,"topic":"Long-horizon planning","total_models":1613,"url":""},{"bench_models":19,"description":"Hughes Hallucination Evaluation Model (Vectara) — lower is better.","github":"https://github.com/vectara/hallucination-leaderboard","has_recent":0,"id":"hhem-v2.1","leaderboard":"https://github.com/vectara/hallucination-leaderboard","lower_is_better":true,"metric":"percent","name":"HHEM v2.1","paper":"","placements":[{"company":"antgroup","family":"Finix","model":"antgroup-finix-s1-32b","place":1,"score":0.6,"version":"Finix S1"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":2,"score":0.7,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro-exp","place":3,"score":0.8,"version":"Gemini 2.0"},{"company":"openai","family":"o3","model":"o3-mini-high","place":4,"score":0.8,"version":"o3"},{"company":"vectara","family":"Mockingbird","model":"vectara-mockingbird-2-echo","place":5,"score":0.9,"version":"Mockingbird 2 Echo"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":6,"score":1.1,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-lite-preview","place":7,"score":1.2,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":8,"score":1.2,"version":"GPT-4.5"},{"company":"zai","family":"GLM","model":"glm-4-9b-chat","place":9,"score":1.3,"version":"GLM 4"},{"company":"tng","family":"DeepSeek-TNG","model":"deepseek-tng-r1t2-chimera","place":10,"score":5.5,"version":"DeepSeek-TNG R1T2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0528","place":11,"score":7.7,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":12,"score":14.3,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":13,"score":88.9,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":14,"score":94.5,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":15,"score":94.5,"version":"Claude 4"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":16,"score":96.5,"version":"openPangu-R"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":17,"score":96.7,"version":"GPT-4.1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":18,"score":97.1,"version":"openPangu-R"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":19,"score":97.8,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":20,"score":98.9,"version":"Kimi K2"}],"stars":"","top_company":"antgroup","top_model":"AntGroup Finix_S1_32b","top_model_family":"Finix","top_model_id":"antgroup-finix-s1-32b","top_model_version":"Finix S1","top_open_source":1,"top_percent":0.6,"topic":"Hallucination detection","total_models":1613,"url":""},{"bench_models":11,"description":"Mathematical reasoning benchmark referenced in recent model cards.","github":"","has_recent":0,"id":"hiddenmath","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HiddenMath","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":1,"score":65.2,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":2,"score":63.5,"version":"Gemini 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":3,"score":60.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":4,"score":54.5,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":5,"score":52,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":6,"score":47.2,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":7,"score":43,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":8,"score":15.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":9,"score":14.8,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":10,"score":10.4,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":11,"score":1.8,"version":"Gemma 2"}],"stars":"","top_company":"google","top_model":"Gemini 2.0 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.0-pro","top_model_version":"Gemini 2.0","top_open_source":0,"top_percent":65.2,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":142,"description":"Challenging LLMs at the frontier of human knowledge.","github":"","has_recent":0,"id":"hle","leaderboard":"https://agi.safe.ai/","lower_is_better":false,"metric":"avg@3","name":"HLE","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-deep-think","place":1,"score":48.4,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":45.8,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":3,"score":44.9,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":4,"score":44.4,"version":"Gemini 3.1"},{"company":"zai","family":"GLM","model":"glm-4.7","place":5,"score":42.8,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":6,"score":41.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"chatgpt-agent","place":7,"score":41.6,"version":"ChatGPT"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":8,"score":40.9,"version":"Seed 1.8"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":9,"score":40.8,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":10,"score":40.8,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":11,"score":40,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":12,"score":40,"version":"Claude 4.6"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-235b","place":13,"score":39.2,"version":"MiroThinker v1.5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.0-72b","place":14,"score":37.7,"version":"MiroThinker v1.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":37.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":37.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":17,"score":37.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":18,"score":37.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":19,"score":37.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":20,"score":36.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":21,"score":36.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":22,"score":36.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":23,"score":35.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":24,"score":35.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":25,"score":35.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":26,"score":35.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":27,"score":34.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":28,"score":34.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":29,"score":34.5,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":30,"score":33.7,"version":"Gemini 3"},{"company":"kuaishou","family":"KAT","model":"kat-coder-pro-v1","place":31,"score":33.4,"version":"KAT-Coder Pro v1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":32,"score":33.2,"version":"Claude 4.6"},{"company":"tongyi","family":"Tongyi","model":"tongyi-deepresearch","place":33,"score":32.9,"version":"Tongyi"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":34,"score":32,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":35,"score":32,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":36,"score":31.8,"version":"MiniMax M2"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":37,"score":31.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":38,"score":31.4,"version":"GPT-5.2"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-30b","place":39,"score":31,"version":"MiroThinker v1.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":40,"score":30.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":41,"score":30.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":42,"score":30.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":43,"score":30.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-pro","place":44,"score":30.7,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":45,"score":30.6,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-5","place":46,"score":30.5,"version":"GLM 5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":47,"score":30.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":48,"score":30.2,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":49,"score":30.1,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":50,"score":30.1,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":51,"score":29.9,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":52,"score":28.7,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":53,"score":28.4,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":54,"score":28.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":55,"score":28.4,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":56,"score":27.2,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":57,"score":27.2,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-deepresearch","place":58,"score":26.9,"version":"Gemini DeepResearch"},{"company":"moonshotai","family":"Kimi","model":"kimi-researcher","place":59,"score":26.9,"version":"Kimi Researcher"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":60,"score":26.9,"version":"Kimi K2"},{"company":"openai","family":"DeepResearch","model":"openai-deepresearch","place":61,"score":26.6,"version":"DeepResearch"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":62,"score":26.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":63,"score":25.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":64,"score":25.7,"version":"GPT-5.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":65,"score":25.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":66,"score":25.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":67,"score":25.1,"version":"DeepSeek V3.2"},{"company":"openai","family":"o3","model":"o3","place":68,"score":24.9,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":69,"score":24.8,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":70,"score":24.5,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":71,"score":24.5,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4","place":72,"score":23.9,"version":"Grok 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":73,"score":23.9,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-codex-high","place":74,"score":23.4,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-high","place":75,"score":22.34,"version":"o3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":76,"score":22.2,"version":"MiniMax M2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":77,"score":22.2,"version":"DeepSeek V3.2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":78,"score":22.1,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":79,"score":21.7,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":80,"score":21.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":81,"score":21.6,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":82,"score":21.4,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.5","place":83,"score":21.2,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":84,"score":21.1,"version":"Gemini 2.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":85,"score":21.1,"version":"MiMo V2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":86,"score":20.3,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3","place":87,"score":20.3,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":88,"score":20.3,"version":"Claude 4"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":89,"score":20,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":90,"score":19.93,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":91,"score":19.8,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":92,"score":19.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":93,"score":18.58,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":94,"score":18.5,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":95,"score":18.5,"version":"GPT-OSS"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":96,"score":18.4,"version":"Grok 4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":97,"score":18.4,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":98,"score":18.1,"version":"Kimi K2"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v0.2-32b","place":99,"score":17.8,"version":"MiroThinker v0.2"},{"company":"openai","family":"o4","model":"o4-mini","place":100,"score":17.7,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":101,"score":17.7,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":102,"score":17.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":103,"score":17.7,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":104,"score":17.6,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":105,"score":17.6,"version":"Grok 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":106,"score":17.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":107,"score":17.3,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":108,"score":17.3,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":109,"score":17.2,"version":"GLM 4.6"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":110,"score":17,"version":"Grok 4"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":111,"score":16.66,"version":"Cogito 671B v2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":112,"score":15.9,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":113,"score":15.9,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":114,"score":15.5,"version":"Nemotron 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":115,"score":15.2,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":116,"score":15,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":117,"score":14.9,"version":"DeepSeek R1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":118,"score":14.8,"version":"GLM 4.5"},{"company":"primeintellect","family":"INTELLECT","model":"intellect-3","place":119,"score":14.6,"version":"INTELLECT"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":120,"score":14.4,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":121,"score":14.23,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":122,"score":13.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":123,"score":13.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":124,"score":13.7,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":125,"score":13.3,"version":"GLM 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":126,"score":12.5,"version":"MiniMax M2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":127,"score":12.5,"version":"MiniMax M2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":128,"score":12.26,"version":"Nemotron"},{"company":"zai","family":"GLM","model":"glm-4.5","place":129,"score":12.2,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":130,"score":12.1,"version":"Gemini 2.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":131,"score":12,"version":"Apriel 1.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":132,"score":12,"version":"Apriel 1.5"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":133,"score":11.9,"version":"Nemotron OpenReasoning"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":134,"score":11.9,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":135,"score":11.7,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":136,"score":11.23,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":137,"score":11.1,"version":"Gemini 2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":138,"score":11.1,"version":"Falcon H1R"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":139,"score":11,"version":"Apriel 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":140,"score":11,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":141,"score":10.9,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":142,"score":10.8,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":143,"score":10.7,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":144,"score":10.6,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":145,"score":10.6,"version":"Nemotron 3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":146,"score":10.5,"version":"EXAONE 4.0"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":147,"score":10.5,"version":"Solar Open"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":148,"score":10.2,"version":"Nemotron 3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":149,"score":10.1,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":150,"score":9.98,"version":"QwQ"},{"company":"mbzuai","family":"K2","model":"k2-think","place":151,"score":9.95,"version":"K2-THINK"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":152,"score":9.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":153,"score":9.8,"version":"GPT-OSS"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":154,"score":9.8,"version":"K2 V2"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":155,"score":9.8,"version":"Apriel 1.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":156,"score":9.8,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":157,"score":9.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":158,"score":9.6,"version":"Claude 4"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":159,"score":9.6,"version":"Magistral 1.2"},{"company":"amazon","family":"Nova","model":"nova-pro","place":160,"score":8.9,"version":"Nova"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":161,"score":8.6,"version":"DeepSeek R1"},{"company":"amazon","family":"Nova","model":"nova-lite","place":162,"score":8.6,"version":"Nova"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":163,"score":8.5,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":164,"score":8.5,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":165,"score":8.4,"version":"DeepSeek V3.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":166,"score":8.4,"version":"MiniMax M1"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":167,"score":8.3,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":168,"score":8.3,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":169,"score":8.2,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":170,"score":8.2,"version":"GPT-5"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":171,"score":7.7,"version":"Mi:dm K 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":172,"score":7.6,"version":"Qwen3"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":173,"score":7.23,"version":"GPT-OSS"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":174,"score":7.2,"version":"MiniMax M1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":175,"score":7.1,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":176,"score":6.8,"version":"GLM 4.5"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":177,"score":6.8,"version":"Llama Nemotron v1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":178,"score":6.7,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":179,"score":6.5,"version":"Nemotron Nano v2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":180,"score":6.3,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":181,"score":6.14,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":182,"score":6.1,"version":"Magistral 1.2"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":183,"score":6,"version":"Ling 2.0"},{"company":"ai21","family":"Jamba","model":"jamba-reasoning-3b","place":184,"score":6,"version":"Jamba Reasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":185,"score":5.9,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":186,"score":5.9,"version":"OLMo 3"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":187,"score":5.9,"version":"Phi-4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":188,"score":5.8,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":189,"score":5.7,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":190,"score":5.6,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":191,"score":5.6,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":192,"score":5.5,"version":"Nemotron OpenReasoning"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":193,"score":5.5,"version":"HyperCLOVA X"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":194,"score":5.4,"version":"GPT-5"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":195,"score":5.2,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":196,"score":5.2,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":197,"score":5.2,"version":"DeepSeek V3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-21b-a3b-pt","place":198,"score":5.1,"version":"ERNIE 4.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":199,"score":5.1,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":200,"score":5.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":201,"score":4.8,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":202,"score":4.8,"version":"Llama 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":203,"score":4.7,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":204,"score":4.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":205,"score":4.4,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":206,"score":4.4,"version":"Nemotron-H"},{"company":"microsoft","family":"Phi","model":"phi-4-mini","place":207,"score":4.2,"version":"Phi 4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":208,"score":4.2,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-large-3","place":209,"score":4.1,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":210,"score":4.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-nothinking-2504","place":211,"score":4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":212,"score":3.7,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":213,"score":3.3,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":214,"score":3.3,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":215,"score":2.7,"version":"GPT-4o"}],"stars":"1085","top_company":"google","top_model":"Gemini 3 Deep Think","top_model_family":"Gemini","top_model_id":"gemini-3-deep-think","top_model_version":"Gemini 3","top_open_source":0,"top_percent":48.4,"topic":"Multi-domain reasoning","total_models":1613,"url":"https://github.com/centerforaisafety/hle"},{"bench_models":15,"description":"Overconfidence rate derived from Humanity's Last Exam evaluations.","github":"","has_recent":0,"id":"hle-overconfidence","leaderboard":"","lower_is_better":true,"metric":"percent","name":"HLE Overconfidence","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":43.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":2,"score":47.8,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":55.1,"version":"Gemini 3"},{"company":"xai","family":"Grok","model":"grok-4","place":4,"score":56.4,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":56.9,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":57.2,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":7,"score":58.8,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":8,"score":60.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":64.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":10,"score":65,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":72,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":12,"score":72.1,"version":"Grok 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":13,"score":80,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":14,"score":80,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":15,"score":83.4,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":43.7,"topic":"Overconfidence / safety","total_models":1613,"url":""},{"bench_models":90,"description":"Humanity's Last Exam benchmark restricted to text-only inputs.","github":"","has_recent":0,"id":"hle-text-only","leaderboard":"https://agi.safe.ai/","lower_is_better":false,"metric":"percent","name":"HLE (Text Only)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":45.8,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":2,"score":44.9,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":41.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":41.7,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4","place":5,"score":41,"version":"Grok 4"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":6,"score":40.9,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":40.3,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":8,"score":40,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":37.5,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":36.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":11,"score":34.5,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":33.3,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":13,"score":33.2,"version":"Claude 4.6"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":14,"score":32.4,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":15,"score":32,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":16,"score":32,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":17,"score":32,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":18,"score":31.7,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":19,"score":30.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":20,"score":29.9,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":21,"score":28.2,"version":"Seed 2.0"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":22,"score":26.9,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":23,"score":26.5,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5-2025-08-07","place":24,"score":26.32,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":25,"score":26.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":26,"score":25.3,"version":"Qwen3.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":27,"score":25.2,"version":"LongCat-Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":28,"score":25.1,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":29,"score":24.4,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":30,"score":24.3,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":31,"score":24.1,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":32,"score":23.9,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":33,"score":23.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":34,"score":22.4,"version":"Qwen3.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":35,"score":22.1,"version":"MiMo V2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":36,"score":22.06,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":37,"score":21.7,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":38,"score":21.6,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3-high","place":39,"score":20.57,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":40,"score":20.3,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":41,"score":19.8,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3-medium","place":42,"score":19.78,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-2025-08","place":43,"score":19.74,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":44,"score":19.4,"version":"GPT-5"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16-high","place":45,"score":18.9,"version":"o4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-exp-03.25","place":46,"score":18.38,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-05-06","place":47,"score":18.38,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":48,"score":18.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":49,"score":18.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":50,"score":17.8,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":51,"score":17.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":52,"score":17.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":53,"score":15.48,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":54,"score":15.43,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":55,"score":14.9,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":56,"score":14.9,"version":"GPT-OSS"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":57,"score":14.53,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":58,"score":14.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":59,"score":14.09,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":60,"score":14.04,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":61,"score":13.7,"version":"Claude 4.5"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":62,"score":13.6,"version":"K-EXAONE"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-high","place":63,"score":13.37,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":64,"score":13.3,"version":"Seed 2.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":65,"score":12.92,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":66,"score":12.88,"version":"DeepSeek V3.1"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":67,"score":12.6,"version":"Nanbeige4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-04-17","place":68,"score":12.58,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":69,"score":11.77,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":70,"score":11.75,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":71,"score":11.33,"version":"Ling 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805-thinking","place":72,"score":11.26,"version":"Claude 4.1"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":73,"score":10.98,"version":"Nanbeige4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-thinking","place":74,"score":10.8,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":75,"score":10.72,"version":"Gemini 2.5"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":76,"score":10.6,"version":"EXAONE 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":77,"score":10.47,"version":"DeepSeek V3.2"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-med","place":78,"score":10.31,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":79,"score":9.73,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5","place":80,"score":9.64,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":81,"score":9.41,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":82,"score":9.31,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":83,"score":8.62,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":84,"score":8.54,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":85,"score":7.89,"version":"Claude 3.7"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":86,"score":7.75,"version":"o1"},{"company":"openai","family":"o1","model":"o1-pro","place":87,"score":7.71,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":88,"score":7.65,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":89,"score":7.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":90,"score":7.37,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":91,"score":7,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":92,"score":6.92,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":93,"score":6.72,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-thinking-01-21","place":94,"score":6.55,"version":"Gemini 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":95,"score":6.26,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.5-preview","place":96,"score":5.8,"version":"GPT-4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":97,"score":5.42,"version":"Claude 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":98,"score":5.34,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":99,"score":5.28,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":100,"score":4.97,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":101,"score":4.68,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":102,"score":4.55,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":103,"score":4.55,"version":"Gemini 1.5"},{"company":"amazon","family":"Nova","model":"nova-micro","place":104,"score":4.41,"version":"Nova"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":105,"score":4.36,"version":"Mistral 3"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20241022","place":106,"score":4.32,"version":"Claude 3.5"},{"company":"amazon","family":"Nova","model":"nova-pro","place":107,"score":4.32,"version":"Nova"},{"company":"amazon","family":"Nova","model":"nova-lite","place":108,"score":3.76,"version":"Nova"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":109,"score":2.32,"version":"GPT-4o"}],"stars":"1085","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":45.8,"topic":"Advanced reasoning","total_models":1613,"url":"https://github.com/centerforaisafety/hle"},{"bench_models":5,"description":"Verified and revised version of Humanity's Last Exam (HLE) with component-wise verification protocol.","github":"","has_recent":0,"id":"hle-verified","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HLE-Verified","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":48,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":43.3,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":38.8,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":4,"score":37.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":5,"score":37.6,"version":"Qwen3.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":48,"topic":"Multi-domain reasoning","total_models":1613,"url":"https://huggingface.co/datasets/skylenage/HLE-Verified"},{"bench_models":3,"description":"Vision-language HLE benchmark.","github":"","has_recent":0,"id":"hle-vl","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HLE-VL","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":36,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":31.5,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":19,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":36,"topic":"Holistic language evaluation (vision-language)","total_models":1613,"url":""},{"bench_models":28,"description":"Humanity's Last Exam benchmark evaluated with tool access.","github":"","has_recent":0,"id":"hle-with-tools","leaderboard":"https://agi.safe.ai/","lower_is_better":false,"metric":"avg@3","name":"HLE (With Tools)","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":53.1,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":53.1,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":3,"score":53,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":4,"score":51.4,"version":"Gemini 3.1"},{"company":"zai","family":"GLM","model":"glm-5","place":5,"score":50.4,"version":"GLM 5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":6,"score":50.2,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":50.2,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":50,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":9,"score":50,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":10,"score":49.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":11,"score":49.8,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":12,"score":49,"version":"Claude 4.6"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":13,"score":48.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":14,"score":48.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":15,"score":47.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":16,"score":47.4,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":17,"score":45.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":18,"score":45.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":19,"score":45.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":20,"score":45.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":21,"score":45.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":22,"score":45.5,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":23,"score":44.9,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":24,"score":43.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":25,"score":43.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":26,"score":43.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":27,"score":43.2,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":28,"score":42.8,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":29,"score":42.7,"version":"GPT-5.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":30,"score":40.8,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":31,"score":40.8,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":32,"score":40.8,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":33,"score":35.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":34,"score":35.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":35,"score":33.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":36,"score":33.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":37,"score":33.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":38,"score":32,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":39,"score":30.4,"version":"GLM 4.6"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":40,"score":19,"version":"GPT-OSS"}],"stars":"1085","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":53.1,"topic":"Tool-augmented reasoning","total_models":1613,"url":"https://github.com/centerforaisafety/hle"},{"bench_models":2,"description":"Harvard–MIT Mathematics Tournament problems.","github":"","has_recent":0,"id":"hmmt","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HMMT","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-pro","place":1,"score":100,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":93.3,"version":"o3"}],"stars":"","top_company":"openai","top_model":"GPT-5 pro","top_model_family":"GPT","top_model_id":"gpt-5-pro","top_model_version":"GPT-5","top_open_source":0,"top_percent":100,"topic":"Math (competition)","total_models":1613,"url":"https://hmmt.org/"},{"bench_models":98,"description":"Harvard–MIT Mathematics Tournament 2025 problems.","github":"","has_recent":0,"id":"hmmt-2025","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HMMT 2025","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":99.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":2,"score":99.6,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":3,"score":99.2,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":4,"score":98.6,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":5,"score":98.1,"version":"GLM 4.7"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":97.5,"version":"Gemini 3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":7,"score":97.5,"version":"LongCat-Flash"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":8,"score":95.1,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":9,"score":94.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":10,"score":93.5,"version":"DeepSeek V3.2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":11,"score":93.4,"version":"LongCat-Flash"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":12,"score":93.3,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":13,"score":93.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":14,"score":93.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":93.3,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":16,"score":92.5,"version":"DeepSeek V3.2"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":17,"score":92.14,"version":"Step3-VL"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":18,"score":91.79,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":19,"score":90.2,"version":"DeepSeek V3.2"},{"company":"xai","family":"Grok","model":"grok-4","place":20,"score":90,"version":"Grok 4"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":21,"score":89.7,"version":"Seed 1.8"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":22,"score":89.4,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":23,"score":89.2,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":24,"score":89.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":25,"score":88.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":26,"score":86.7,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":27,"score":86.1,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":28,"score":86.1,"version":"DeepSeek V3.1"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":29,"score":84.53,"version":"Ring"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":30,"score":84.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":31,"score":83.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":32,"score":83.9,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":33,"score":83.6,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":34,"score":83.6,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":35,"score":83.54,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":36,"score":82.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":37,"score":81.88,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":38,"score":81.7,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3-high","place":39,"score":80.8,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":40,"score":79.2,"version":"Claude 4.5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":41,"score":78.18,"version":"Step3-VL"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":42,"score":74.6,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":43,"score":74.18,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":44,"score":73.9,"version":"Qwen3 Next"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":45,"score":73.8,"version":"Nemotron OpenReasoning"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":46,"score":73.3,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":47,"score":71.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":48,"score":71.4,"version":"Qwen3"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":49,"score":71.4,"version":"K2 V2"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":50,"score":71.2,"version":"Nemotron OpenReasoning"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":51,"score":69.38,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":52,"score":67.71,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":53,"score":67.6,"version":"Qwen3 VL"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":54,"score":67.29,"version":"Nemotron"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":55,"score":66.7,"version":"Claude 4.5"},{"company":"mbzuai","family":"K2","model":"k2-think","place":56,"score":66.46,"version":"K2-THINK"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":57,"score":65.68,"version":"Gemini 2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":58,"score":64.9,"version":"Falcon H1R"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":59,"score":64.8,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":60,"score":64.2,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":61,"score":63.5,"version":"Nemotron OpenReasoning"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":62,"score":63.5,"version":"MiniMax M2"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":63,"score":61,"version":"Apriel 1.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":64,"score":57.29,"version":"GLM 4.6V"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":65,"score":54.3,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":66,"score":54.1,"version":"Qwen3 Next"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":67,"score":53,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":68,"score":51.5,"version":"Qwen3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":69,"score":51.46,"version":"QwQ"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":70,"score":51.3,"version":"Seed 1.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":71,"score":50.83,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":72,"score":50.6,"version":"Qwen3 VL"},{"company":"weibo","family":"VibeThinker","model":"vibethinker-1.5b","place":73,"score":50.4,"version":"VibeThinker"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":74,"score":50.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":75,"score":49.8,"version":"Qwen3"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":76,"score":47.7,"version":"Phi-4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":77,"score":47.08,"version":"DeepSeek R1"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":78,"score":45.6,"version":"K2 V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":79,"score":44.5,"version":"Kimi Linear"},{"company":"microsoft","family":"Phi","model":"phi-4-reasoning","place":80,"score":43.8,"version":"Phi 4"},{"company":"mistral","family":"Magistral","model":"magistral-small-2506","place":81,"score":43.5,"version":"Magistral 2506"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":82,"score":43.3,"version":"OLMo 3"},{"company":"nvidia","family":"AceReason","model":"acereason-nemotron-1.1","place":83,"score":42.9,"version":"1.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0120","place":84,"score":41.7,"version":"0120"},{"company":"amazon","family":"Nemotron","model":"questa-nemotron-1.5b","place":85,"score":41.67,"version":"QuestA Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":86,"score":41,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":87,"score":38.8,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":88,"score":38.8,"version":"Kimi K2"},{"company":"xiaomi","family":"MiMo","model":"mimo-7b-rl","place":89,"score":38.3,"version":"MiMo 7B"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":90,"score":35.8,"version":"Ling 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":91,"score":34.7,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":92,"score":34.2,"version":"Nemotron-H"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":93,"score":34,"version":"EXAONE 4.0"},{"company":"deepseek","family":"DeepSeek-R1","model":"deepseek-r1-distill-32b","place":94,"score":33,"version":"32B"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":95,"score":32.5,"version":"Qwen3 VL"},{"company":"KunLun","family":"Skywork OR1","model":"skywork-or1-7b","place":96,"score":32,"version":"Skywork OR1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-1.5b","place":97,"score":31.56,"version":"Nemotron"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":98,"score":31.5,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":99,"score":31,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":100,"score":30.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":101,"score":29.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":102,"score":27.5,"version":"DeepSeek V3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":103,"score":27.3,"version":"EXAONE Deep"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":104,"score":26,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":105,"score":22.19,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":106,"score":21.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":107,"score":19.4,"version":"GPT-4.1"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":108,"score":19,"version":"K2 V2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":109,"score":15.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":110,"score":15.9,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":111,"score":11.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":112,"score":11.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-nothinking-2504","place":113,"score":11.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":114,"score":9.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":115,"score":9.79,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":116,"score":7,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-21b-a3b-pt","place":117,"score":6.9,"version":"ERNIE 4.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":99.8,"topic":"Math (competition)","total_models":1613,"url":""},{"bench_models":40,"description":"Harvard–MIT Mathematics Tournament February 2025 problems.","github":"","has_recent":0,"id":"hmmt-feb-2025","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HMMT Feb 2025","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":100,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":100,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":3,"score":99.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":4,"score":99.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":99.4,"version":"GPT-5.2"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":6,"score":98.4,"version":"Step 3.5 Flash"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":7,"score":98,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":97.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":97.3,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":10,"score":97.3,"version":"Seed 2.0"},{"company":"zai","family":"GLM","model":"glm-4.7","place":11,"score":97.1,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":12,"score":96.3,"version":"GPT-5.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":13,"score":95.4,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":14,"score":95.4,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":15,"score":94.8,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":16,"score":93.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":17,"score":92.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":18,"score":92.9,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":19,"score":92.5,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":20,"score":92.5,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":21,"score":92,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":22,"score":91.4,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":23,"score":90,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":24,"score":90,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":25,"score":89.4,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":26,"score":89.2,"version":"GLM 4.6"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":27,"score":89.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":28,"score":89,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":29,"score":88.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":30,"score":85.1,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":31,"score":84.4,"version":"MiMo V2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":32,"score":80,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":33,"score":79.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":34,"score":79.2,"version":"Claude 4.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":35,"score":74.8,"version":"openPangu-R"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":36,"score":73.3,"version":"Solar Open"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":37,"score":71,"version":"MiniMax M2.1"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":38,"score":70,"version":"Seed 2.0"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":39,"score":66.7,"version":"GLM 4.5"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":40,"score":63.3,"version":"GPT-OSS"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":41,"score":45.4,"version":"openPangu-R"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":100,"topic":"Math (competition)","total_models":1613,"url":"https://hmmt.org/"},{"bench_models":47,"description":"Harvard–MIT Mathematics Tournament November 2025 problems.","github":"","has_recent":0,"id":"hmmt-nov-2025","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HMMT Nov 2025","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":100,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":100,"version":"GPT-5.2"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5-heavy-thinking","place":3,"score":98.33,"version":"Ring 2.5"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5","place":4,"score":97.34,"version":"Ring 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":5,"score":96.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":6,"score":96.7,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":7,"score":94.7,"version":"Qwen3"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":8,"score":94,"version":"Step 3.5 Flash"},{"company":"zai","family":"GLM","model":"glm-4.7","place":9,"score":93.5,"version":"GLM 4.7"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":93.3,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":93.3,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":12,"score":93.3,"version":"Seed 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":13,"score":92.7,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":14,"score":91.51,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":15,"score":91.1,"version":"Kimi K2.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":16,"score":91,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":17,"score":90.3,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":18,"score":90.2,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":19,"score":90.2,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":20,"score":90,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":21,"score":89.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":22,"score":89.5,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":23,"score":89.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":24,"score":89.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":25,"score":89.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":26,"score":88.8,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":27,"score":87.7,"version":"GLM 4.6"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":28,"score":86.8,"version":"K-EXAONE"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":29,"score":86.7,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":30,"score":84.9,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":31,"score":84.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":32,"score":81.7,"version":"Claude 4.5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":33,"score":80.21,"version":"Ling 2.5"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":34,"score":80,"version":"Solar Open"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":35,"score":80,"version":"Seed 2.0"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":36,"score":78.1,"version":"EXAONE 4.0"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":37,"score":77.92,"version":"Nanbeige4.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":38,"score":74.3,"version":"MiniMax M2.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":39,"score":73.3,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":40,"score":71.25,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":41,"score":70,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":42,"score":68.33,"version":"Qwen3"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":43,"score":66.7,"version":"GPT-OSS"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":44,"score":66.67,"version":"Nanbeige4"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":45,"score":66.46,"version":"Ling 1T"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":46,"score":61.2,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":47,"score":57.08,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":48,"score":56.67,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":49,"score":53.44,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":50,"score":53.18,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":51,"score":48.33,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":100,"topic":"Math (competition)","total_models":1613,"url":"https://hmmt.org/"},{"bench_models":7,"description":"Explainable multi-hop QA with supporting facts.","github":"","has_recent":0,"id":"hotpotqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HotpotQA","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":1,"score":64,"version":"Qwen3"},{"company":"pleias","family":"Baguettotron","model":"baguettotron-321m","place":2,"score":51,"version":"Baguettotron"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":3,"score":25,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4-300m","place":4,"score":25,"version":"Granite 4"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":5,"score":23,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4-h-300m","place":6,"score":21,"version":"Granite 4"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-instruct","place":7,"score":16,"version":"SmolLM2"}],"stars":"","top_company":"qwen","top_model":"Qwen 3 0.6B","top_model_family":"Qwen","top_model_id":"qwen-3-0.6b","top_model_version":"Qwen3","top_open_source":1,"top_percent":64,"topic":"Multi-hop QA","total_models":1613,"url":"https://hotpotqa.github.io/"},{"bench_models":12,"description":"Hallucination robustness benchmark with 4K token contexts.","github":"","has_recent":0,"id":"hrbench4k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HRBench 4K","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":1,"score":89.5,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":86.9,"version":"Gemini 2.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":3,"score":84,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":4,"score":82.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":5,"score":81.2,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":6,"score":79.9,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":7,"score":78.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":8,"score":78.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":9,"score":77.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":10,"score":75.1,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":11,"score":74.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":12,"score":69.1,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":13,"score":67.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":14,"score":67.6,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":15,"score":50.6,"version":"Claude 4"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-30B-A3B Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-30b-a3b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":89.5,"topic":"Hallucination robustness","total_models":1613,"url":""},{"bench_models":12,"description":"Hallucination robustness benchmark with 8K token contexts.","github":"","has_recent":0,"id":"hrbench8k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HRBench 8K","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":84,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":2,"score":82.5,"version":"Qwen3 VL"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":3,"score":80.4,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":4,"score":80,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":5,"score":79.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":6,"score":77.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":7,"score":75.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":8,"score":74.6,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":9,"score":74.3,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":10,"score":72.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":71.3,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":12,"score":71,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":13,"score":68.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":14,"score":68,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":15,"score":66.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":16,"score":40.5,"version":"Claude 4"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":84,"topic":"Hallucination robustness","total_models":1613,"url":""},{"bench_models":5,"description":"8k-question Korean reasoning and knowledge benchmark.","github":"","has_recent":0,"id":"hrm8k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HRM8K","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":1,"score":92,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":2,"score":91.6,"version":"GPT-OSS"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":3,"score":90.9,"version":"K-EXAONE"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":90.6,"version":"DeepSeek V3.2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":5,"score":89.4,"version":"EXAONE 4.0"}],"stars":"","top_company":"qwen","top_model":"Qwen3-235B-A22B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-235b-a22b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":92,"topic":"Korean reasoning","total_models":1613,"url":""},{"bench_models":258,"description":"Python synthesis problems evaluated by unit tests.","github":"https://github.com/openai/human-eval","has_recent":0,"id":"humaneval","leaderboard":"https://llmdb.com/benchmarks/humaneval","lower_is_better":false,"metric":"percent","name":"HumanEval","paper":"https://arxiv.org/abs/2107.03374","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":1,"score":100,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":2,"score":98.8,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":98.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":98.8,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":5,"score":98.2,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":6,"score":97.6,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":7,"score":97.6,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":8,"score":97.6,"version":"IQuest Coder V1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":9,"score":97.2,"version":"Tulu 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":10,"score":97,"version":"GPT-4o"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":11,"score":96.95,"version":"Ministral 3"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":12,"score":96.34,"version":"Falcon H1R"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":13,"score":96.34,"version":"JoyAI-LLM"},{"company":"openai","family":"o1","model":"o1-preview","place":14,"score":96.3,"version":"o1"},{"company":"openai","family":"o1","model":"o1-mini","place":15,"score":96.3,"version":"o1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":16,"score":96.3,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":17,"score":96.3,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":18,"score":96.3,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":19,"score":96.3,"version":"IQuest Coder V1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":20,"score":95.9,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":21,"score":95.9,"version":"Llama 3.1"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":22,"score":95.9,"version":"Youtu-LLM"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":23,"score":95.7,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":24,"score":95.4,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":25,"score":95.12,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":26,"score":95.12,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":27,"score":94.6,"version":"DeepSeek V3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":28,"score":94.51,"version":"LLaDA 2.0"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":29,"score":94.51,"version":"HY"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":30,"score":94.5,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":31,"score":94.5,"version":"LLaDA 2.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":32,"score":94.5,"version":"Kimi K2"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":33,"score":94.1,"version":"Llama 3.1 Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":34,"score":94,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":35,"score":93.9,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":36,"score":93.9,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":37,"score":93.9,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":38,"score":93.9,"version":"Nemotron Nano V2"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":39,"score":93.6,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":40,"score":93.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":41,"score":93.3,"version":"Qwen2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":42,"score":93.3,"version":"Kimi Dev"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":43,"score":93.29,"version":"Qwen3"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":44,"score":93.29,"version":"HY"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":45,"score":93.2,"version":"Motif 2"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":46,"score":93.2,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":47,"score":93.1,"version":"Qwen2.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":48,"score":92.9,"version":"Tulu 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":49,"score":92.7,"version":"GPT-4o"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":50,"score":92.4,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":51,"score":92.4,"version":"Tulu 3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":52,"score":92.3,"version":"Hermes 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":53,"score":92.1,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":54,"score":92.1,"version":"Mistral 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":55,"score":91.5,"version":"DeepSeek V3"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":56,"score":91.5,"version":"K2 V2"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":57,"score":91.5,"version":"K2 V2"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":58,"score":91.46,"version":"MiniCPM 4.1"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":59,"score":91,"version":"Ministral"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":60,"score":90.9,"version":"KAT"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":61,"score":90.2,"version":"DeepSeek V2.5"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":62,"score":90.2,"version":"GPT-4"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":63,"score":89.73,"version":"Granite 3.3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":64,"score":89.6,"version":"MiniMax M2"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":65,"score":89.6,"version":"Hermes 3"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":66,"score":89.02,"version":"HY"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":67,"score":88.4,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":68,"score":88.4,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4","place":69,"score":88.4,"version":"GPT-4"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":70,"score":88.4,"version":"KAT"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":71,"score":88.4,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":72,"score":88.1,"version":"DeepSeek R1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":73,"score":88,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":74,"score":87.8,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":75,"score":87.2,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":76,"score":87.2,"version":"Qwen2.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":77,"score":87.2,"version":"GLM 4.7"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":78,"score":86.3,"version":"Llama 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":79,"score":86.2,"version":"Tulu 3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":80,"score":86,"version":"Ling 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":81,"score":85.98,"version":"Ling 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":82,"score":85.4,"version":"DeepSeek Coder V2"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":83,"score":85.4,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":84,"score":85.4,"version":"Qwen2.5"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":85,"score":85.37,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":86,"score":84.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":87,"score":84.8,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":88,"score":83.9,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":89,"score":83.9,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b-chat","place":90,"score":83.5,"version":"CodeQwen1.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":91,"score":82.9,"version":"Claude 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":92,"score":82.41,"version":"Granite 4.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":93,"score":82.3,"version":"Gemini 1.5"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":94,"score":82.3,"version":"K2 V2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":95,"score":82,"version":"Granite 4.0"},{"company":"map","family":"OpenCoder","model":"opencoder-8b-instruct","place":96,"score":81.7,"version":"OpenCoder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":97,"score":81.1,"version":"DeepSeek Coder"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":98,"score":81.1,"version":"Codestral"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":99,"score":81.1,"version":"DeepSeek Coder V2"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":100,"score":81.1,"version":"Seed Coder"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":101,"score":80.51,"version":"Granite 3.3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":102,"score":80.5,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":103,"score":80.49,"version":"LLaDA 2.0"},{"company":"tencent","family":"WeDLM","model":"wedlm-8b-instruct","place":104,"score":80.49,"version":"WeDLM"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":105,"score":80,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":106,"score":80,"version":"Granite 4.0"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-33b-v1.1","place":107,"score":79.9,"version":"WizardCoder V1.1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":108,"score":79.9,"version":"SmollM3"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-33b","place":109,"score":79.3,"version":"OpenCodeInterpreter DS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":110,"score":79.3,"version":"Qwen2.5"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":111,"score":78.7,"version":"Ouro"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":112,"score":78.05,"version":"Nemotron 3"},{"company":"meta","family":"Llama3-70B-instruct","model":"llama3-70b-instruct","place":113,"score":77.4,"version":"Llama3-70B-instruct"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-6.7b","place":114,"score":77.4,"version":"OpenCodeInterpreter DS"},{"company":"speechless","family":"speechless-codellama-34B-v2.0","model":"speechless-codellama-34b-v2.0","place":115,"score":77.4,"version":"speechless-codellama-34B-v2.0"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":116,"score":77.4,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":117,"score":77.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo","place":118,"score":76.8,"version":"GPT-3.5"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":119,"score":76.8,"version":"Magicoder-S-DS-6.7B"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":120,"score":76.8,"version":"Claude 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b-instruct-v0.1","place":121,"score":76.2,"version":"Mixtral v0.1"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":122,"score":76.2,"version":"Llama 3"},{"company":"artigenz","family":"Artigenz-Coder-DS-6.7B","model":"artigenz-coder-ds-6.7b","place":123,"score":75.6,"version":"Artigenz Coder DS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-7b-instruct-v1.5","place":124,"score":75.6,"version":"DeepSeek Coder"},{"company":"xwin","family":"XwinCoder-34B","model":"xwincoder-34b","place":125,"score":75.6,"version":"XwinCoder-34B"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":126,"score":75.59,"version":"Kanana 1.5"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":127,"score":75.4,"version":"Magpie"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":128,"score":75.29,"version":"Kanana 2"},{"company":"mistral","family":"Codestral","model":"codestral-mamba","place":129,"score":75,"version":"Codestral"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":130,"score":75,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":131,"score":75,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":132,"score":74.4,"version":"DeepSeek Coder"},{"company":"code","family":"code-millenials-34B","model":"code-millenials-34b","place":133,"score":74.4,"version":"code-millenials-34B"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":134,"score":74.4,"version":"Ouro"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":135,"score":74.39,"version":"GLM 4.7"},{"company":"huggingface","family":"starchat2-15b-v0.1","model":"starchat2-15b-v0.1","place":136,"score":73.8,"version":"starchat2-15b-v0.1"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":137,"score":73.58,"version":"Mistral 3.2"},{"company":"openai","family":"GPT","model":"gpt-3.5","place":138,"score":73.2,"version":"GPT-3"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-python-34b-v1.0","place":139,"score":73.2,"version":"WizardCoder V1.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":140,"score":73.17,"version":"Gemma 3"},{"company":"openchat","family":"OpenChat-3.5-7B-0106","model":"openchat-3.5-7b-0106","place":141,"score":72.6,"version":"OpenChat-3.5-7B-0106"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-instruct","place":142,"score":71.95,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":143,"score":71.7,"version":"Gemma 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":144,"score":71.34,"version":"Kanana 2"},{"company":"phind","family":"Phind-CodeLlama-34B-v2","model":"phind-codellama-34b-v2","place":145,"score":71.3,"version":"Phind-CodeLlama-34B-v2"},{"company":"speechless","family":"speechless-coder-ds-6.7B","model":"speechless-coder-ds-6.7b","place":146,"score":71.3,"version":"speechless-coder-ds-6.7B"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":147,"score":71.14,"version":"Llama 3.3"},{"company":"magicoder","family":"Magicoder-S-CL-7B","model":"magicoder-s-cl-7b","place":148,"score":70.7,"version":"Magicoder-S-CL-7B"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet","place":149,"score":70.7,"version":"Claude 3"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":150,"score":69.5,"version":"Mistral"},{"company":"anthropic","family":"claude-2","model":"claude-2","place":151,"score":69.5,"version":"claude-2 (Mar 2024)"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":152,"score":68.9,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-72b-chat","place":153,"score":68.3,"version":"Qwen1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":154,"score":68.3,"version":"Gemini 1.5"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b-instruct-v0.1","place":155,"score":67.7,"version":"StarCoder2"},{"company":"bigcode","family":"speechless-starcoder2-15b","model":"speechless-starcoder2-15b","place":156,"score":67.1,"version":"speechless-starcoder2-15b"},{"company":"meta","family":"CodeLlama","model":"codellama-70b","place":157,"score":67.1,"version":"CodeLlama"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":158,"score":67.07,"version":"HY"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":159,"score":66.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-instruct","place":160,"score":65.9,"version":"DeepSeek Coder"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":161,"score":65.9,"version":"DeepSeek Math"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":162,"score":65.9,"version":"Motif 2"},{"company":"code","family":"Code","model":"code-290k-6.7b-instruct","place":163,"score":64.6,"version":"Code 290k"},{"company":"microsoft","family":"Phi-3-mini-4k-instruct","model":"phi-3-mini-4k-instruct","place":164,"score":64.6,"version":"Phi-3-mini-4k-instruct"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":165,"score":64,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-1.0-pro","place":166,"score":63.4,"version":"Gemini 1.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":167,"score":62.8,"version":"Qwen2.5"},{"company":"meta","family":"Llama3-8B-instruct","model":"llama3-8b-instruct","place":168,"score":61.6,"version":"Llama3-8B-instruct"},{"company":"google","family":"CodeGemma","model":"codegemma-1.1-7b","place":169,"score":61,"version":"CodeGemma"},{"company":"google","family":"Codegemma","model":"codegemma-7b-it","place":170,"score":60.4,"version":"Codegemma"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":171,"score":59.8,"version":"MobileLLM"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":172,"score":59.2,"version":"Nemotron Nano v2"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":173,"score":57.9,"version":"Dream 7B"},{"company":"anthropic","family":"claude-instant-1","model":"claude-instant-1-mar-2024","place":174,"score":57.3,"version":"claude-instant-1 (Mar 2024)"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-15b-v1.0","place":175,"score":56.7,"version":"WizardCoder V1.0"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":176,"score":56.7,"version":"Dream"},{"company":"code","family":"Code-13B","model":"code-13b","place":177,"score":56.1,"version":"Code-13B"},{"company":"bigcode","family":"speechless-starcoder2-7b","model":"speechless-starcoder2-7b","place":178,"score":56.1,"version":"speechless-starcoder2-7b"},{"company":"code","family":"Code-33B","model":"code-33b","place":179,"score":54.9,"version":"Code-33B"},{"company":"speechless","family":"speechless-coding-7B-16k-tora","model":"speechless-coding-7b-16k-tora","place":180,"score":54.9,"version":"speechless-coding-7B-16k-tora"},{"company":"openhermes","family":"OpenHermes-2.5-Code-290k-13B","model":"openhermes-2.5-code-290k-13b","place":181,"score":54.3,"version":"OpenHermes-2.5-Code-290k-13B"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-instruct","place":182,"score":54.3,"version":"CoDA"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":183,"score":54.3,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":184,"score":54.3,"version":"Qwen2.5"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":185,"score":53.86,"version":"EuroLLM"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":186,"score":53.32,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":187,"score":52.4,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":188,"score":52.03,"version":"OLMo 2"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":189,"score":51.8,"version":"CodeLlama-34B"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b","place":190,"score":51.8,"version":"CodeQwen1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-base","place":191,"score":51.2,"version":"DeepSeek Coder"},{"company":"microsoft","family":"WizardCoder-Python-7B-V1.0","model":"wizardcoder-python-7b-v1.0","place":192,"score":50.6,"version":"WizardCoder-Python-7B-V1.0"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":193,"score":50.6,"version":"Llama 3.1"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":194,"score":50,"version":"K2 V2"},{"company":"microsoft","family":"phi-2-2.7B","model":"phi-2-2.7b","place":195,"score":49.4,"version":"phi-2-2.7B"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b","place":196,"score":49,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":197,"score":48.8,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":198,"score":48.8,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-codealpaca-7b","place":199,"score":48.2,"version":"Mistral Codealpaca"},{"company":"speechless","family":"speechless-code-mistral-7B-v1.0","model":"speechless-code-mistral-7b-v1.0","place":200,"score":48.2,"version":"speechless-code-mistral-7B-v1.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-base","place":201,"score":47.6,"version":"DeepSeek Coder"},{"company":"mistral","family":"MistralHermes","model":"mistralhermes-codepro-7b-v1","place":202,"score":47.6,"version":"MistralHermes CodePro v1"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b","place":203,"score":46.3,"version":"StarCoder2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":204,"score":46.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":205,"score":45.7,"version":"Gemma 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b-instruct-v0.1","place":206,"score":45.1,"version":"Mixtral v0.1"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":207,"score":45.1,"version":"Mixtral"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":208,"score":44.51,"version":"Apertus"},{"company":"google","family":"Codegemma","model":"codegemma-7b","place":209,"score":44.5,"version":"codegemma-7b"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":210,"score":43.3,"version":"Solar Mini"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":211,"score":43.3,"version":"CodeLlama-34B"},{"company":"codellama","family":"CodeLlama-13B","model":"codellama-13b","place":212,"score":42.7,"version":"CodeLlama-13B"},{"company":"google","family":"Gemma","model":"gemma-1.1-7b-it","place":213,"score":42.7,"version":"gemma-1.1"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":214,"score":42.7,"version":"Marin 32B"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0-2","place":215,"score":42.1,"version":"Mistral 0.2"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":216,"score":42.1,"version":"Llama 3.0"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":217,"score":41.5,"version":"Gemma 3"},{"company":"alephalpha","family":"TFree","model":"tfree-hat-pretrained-7b-base","place":218,"score":41.1,"version":"TFree Hat"},{"company":"allenai","family":"Bolmo","model":"bolmo-7b","place":219,"score":40.6,"version":"Bolmo 7B"},{"company":"xdan","family":"xDAN-L1-Chat-RL-v1-7B","model":"xdan-l1-chat-rl-v1-7b","place":220,"score":40.2,"version":"xDAN-L1-Chat-RL-v1-7B"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":221,"score":40.2,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":222,"score":38.4,"version":"Llama 3.1"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":223,"score":37.8,"version":"CodeLlama-7B"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":224,"score":37.8,"version":"Llama 3.2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":225,"score":36,"version":"OLMo 3"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-7b","place":226,"score":35.4,"version":"StarCoder2"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":227,"score":35.4,"version":"gemma-7b"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":228,"score":35.4,"version":"LLaDA"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":229,"score":34.8,"version":"Gemma 3"},{"company":"hku","family":"EvaByte","model":"evabyte-6.5b","place":230,"score":34.7,"version":"EvaByte 6.5B"},{"company":"bigcode","family":"StarCoder-15B","model":"starcoder-15b","place":231,"score":34.1,"version":"StarCoder"},{"company":"meta","family":"Llama3-8B-base","model":"llama3-8b-base","place":232,"score":33.5,"version":"Llama3-8B-base"},{"company":"salesforce","family":"CodeGen","model":"codegen-16b","place":233,"score":32.9,"version":"CodeGen"},{"company":"python","family":"Python-Code-13B","model":"python-code-13b","place":234,"score":32.9,"version":"Python-Code-13B"},{"company":"salesforce","family":"Codet5plus","model":"codet5plus-16b","place":235,"score":31.7,"version":"Codet5plus"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-3b","place":236,"score":31.7,"version":"StarCoder2"},{"company":"meta","family":"BLT","model":"blt-7b","place":237,"score":31.5,"version":"BLT 7B"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":238,"score":31.1,"version":"CodeLlama-7B"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":239,"score":30.5,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":240,"score":29.9,"version":"Llama 3.2"},{"company":"salesforce","family":"CodeGen","model":"codegen-6b","place":241,"score":29.3,"version":"CodeGen"},{"company":"salesforce","family":"Codet5plus","model":"codet5plus-6b","place":242,"score":29.3,"version":"Codet5plus"},{"company":"stable","family":"stable-code-3B","model":"stable-code-3b","place":243,"score":29.3,"version":"stable-code-3B"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-base","place":244,"score":29.3,"version":"CoDA"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":245,"score":29.3,"version":"Llama 2"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":246,"score":29.3,"version":"Marin 32B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-base","place":247,"score":28.7,"version":"DeepSeek Coder"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":248,"score":28.7,"version":"Mistral"},{"company":"google","family":"Gemma","model":"gemma-7b-it","place":249,"score":28.7,"version":"Gemma"},{"company":"google","family":"Codegemma","model":"codegemma-2b","place":250,"score":26.8,"version":"codegemma-2b"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":251,"score":26.22,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":252,"score":26.2,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":253,"score":25,"version":"Llama 1"},{"company":"salesforce","family":"CodeGen","model":"codegen-2b","place":254,"score":24.4,"version":"CodeGen"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-7b","place":255,"score":24.4,"version":"StarCoderBase"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":256,"score":23.8,"version":"OLMo 2"},{"company":"google","family":"gemma-1.1-2b-it","model":"gemma-1.1-2b-it","place":257,"score":22.6,"version":"gemma-1.1"},{"company":"salesforce","family":"CodeGen","model":"codegen2-16b","place":258,"score":19.5,"version":"CodeGen2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":259,"score":19.5,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":260,"score":18.9,"version":"Llama 2"},{"company":"salesforce","family":"CodeGen","model":"codegen2-7b","place":261,"score":18.3,"version":"CodeGen2"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-3b","place":262,"score":17.7,"version":"StarCoderBase"},{"company":"google","family":"Gemma","model":"gemma-2b-it","place":263,"score":17.7,"version":"Gemma"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":264,"score":17.6,"version":"Gemma 3"},{"company":"lmsys","family":"Vicuna-13B","model":"vicuna-13b","place":265,"score":17.1,"version":"Vicuna-13B"},{"company":"salesforce","family":"CodeGen","model":"codegen2-3b","place":266,"score":15.9,"version":"CodeGen2"},{"company":"ai21","family":"InCoder","model":"incoder-6.7b","place":267,"score":15.9,"version":"InCoder"},{"company":"bigscience","family":"BLOOM","model":"bloom-176b","place":268,"score":15.5,"version":"176B"},{"company":"bigcode","family":"SantaCoder","model":"santacoder-1.1b","place":269,"score":14.6,"version":"SantaCoder"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-1b","place":270,"score":14.6,"version":"StarCoderBase"},{"company":"openai","family":"GPT","model":"gpt-j-6b","place":271,"score":12.2,"version":"GPT-J"},{"company":"ai21","family":"InCoder","model":"incoder-1.3b","place":272,"score":12.2,"version":"InCoder"},{"company":"lmsys","family":"Vicuna-7B","model":"vicuna-7b","place":273,"score":11.6,"version":"Vicuna-7B"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":274,"score":11.6,"version":"Llama 2"},{"company":"openai","family":"GPT","model":"gpt-neo-2.7b","place":275,"score":7.9,"version":"GPT-Neo"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":276,"score":6.7,"version":"Gemma 3"},{"company":"cmu","family":"PolyCoder-2.7B","model":"polycoder-2.7b","place":277,"score":6.1,"version":"PolyCoder-2.7B"},{"company":"stabilityai","family":"StableLM-7B","model":"stablelm-7b","place":278,"score":2.4,"version":"StableLM-7B"},{"company":"zysecai","family":"Zyte","model":"zyte-1b","place":279,"score":2.4,"version":"Zyte"}],"stars":"2916","top_company":"google","top_model":"Gemini 3 Pro Preview","top_model_family":"Gemini","top_model_id":"gemini-3-pro-preview","top_model_version":"Gemini 3","top_open_source":0,"top_percent":100,"topic":"Code generation","total_models":1613,"url":"https://github.com/openai/human-eval"},{"bench_models":269,"description":"Extended HumanEval with more tests.","github":"","has_recent":0,"id":"humaneval-plus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HumanEval+","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":1,"score":94.51,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":2,"score":94.5,"version":"Gemini 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":3,"score":93.9,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":4,"score":93.3,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":5,"score":93.3,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":93.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":93.3,"version":"Claude 4.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":8,"score":92.9,"version":"Tulu 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":9,"score":92.7,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":10,"score":92.7,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":11,"score":92.7,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":12,"score":92.3,"version":"DeepSeek R1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":13,"score":92.1,"version":"ERNIE 4.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":14,"score":92.1,"version":"GPT-4.1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":15,"score":92.1,"version":"GLM 4.6"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":16,"score":91.6,"version":"OLMo 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":17,"score":91.6,"version":"DeepSeek V3"},{"company":"allenai","family":"OLMo","model":"olmo-3.1-think-32b","place":18,"score":91.5,"version":"OLMo 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":19,"score":91.5,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":20,"score":91.5,"version":"IQuest Coder V1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":21,"score":91.4,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":22,"score":91.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":23,"score":90.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":24,"score":90.6,"version":"Qwen3 VL"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":25,"score":90.3,"version":"Llama 3.1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":26,"score":90.2,"version":"IQuest Coder V1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":27,"score":90,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":28,"score":89.7,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":29,"score":89.63,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":30,"score":89.63,"version":"LLaDA 2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":31,"score":89.6,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":32,"score":89.6,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":33,"score":89.6,"version":"Kimi K2"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":34,"score":89.5,"version":"Llama 3.1"},{"company":"openai","family":"o1","model":"o1-preview","place":35,"score":89,"version":"o1"},{"company":"openai","family":"o1","model":"o1-mini","place":36,"score":89,"version":"o1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":37,"score":89,"version":"GLM 4.6"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":38,"score":89,"version":"Youtu-LLM"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":39,"score":88.5,"version":"Ministral"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":40,"score":88.41,"version":"LLaDA 2.0"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":41,"score":88.4,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":42,"score":88.4,"version":"DeepSeek V3.2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":43,"score":88,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":44,"score":87.88,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":45,"score":87.8,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":46,"score":87.8,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":47,"score":87.58,"version":"Ling 2.0"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":48,"score":87.3,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":49,"score":87.2,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":50,"score":87.2,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":51,"score":87.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":52,"score":87.2,"version":"Qwen3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":53,"score":86.9,"version":"Hermes 3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":54,"score":86.6,"version":"GPT-4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":55,"score":86.6,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":56,"score":86.6,"version":"Qwen2.5"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":57,"score":86.6,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":58,"score":86.59,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":59,"score":86.09,"version":"Granite 3.3"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":60,"score":86,"version":"Kimi Dev"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":61,"score":85.9,"version":"Hermes 3"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":62,"score":85.79,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":63,"score":85.72,"version":"Granite 3.2"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":64,"score":85.5,"version":"Llama 3.1 Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":65,"score":85.37,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":66,"score":84.8,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":67,"score":84.8,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":68,"score":84.1,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":69,"score":83.5,"version":"DeepSeek V2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":70,"score":83.5,"version":"GPT-4o"},{"company":"essential","family":"Rnj","model":"rnj-1","place":71,"score":83.5,"version":"Rnj 1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":72,"score":83,"version":"Granite 4.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":73,"score":82.93,"version":"LLaDA 2.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":74,"score":82.9,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":75,"score":82.6,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":76,"score":82.5,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":77,"score":82.32,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":78,"score":82.32,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":79,"score":82.3,"version":"DeepSeek Coder V2"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":80,"score":81.71,"version":"Falcon H1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":81,"score":81.71,"version":"LLaDA 2.0"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":82,"score":81.7,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":83,"score":81.7,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":84,"score":81.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":85,"score":81.7,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":86,"score":81.7,"version":"Qwen2.5"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":87,"score":81.7,"version":"KAT"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":88,"score":81.4,"version":"Tulu 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":89,"score":81.1,"version":"Falcon H1"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":90,"score":81.1,"version":"Kanana 2"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":91,"score":80.5,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":92,"score":80.49,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":93,"score":80.49,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":94,"score":80.03,"version":"Ling 2.0"},{"company":"skt","family":"A.X","model":"ax-k1","place":95,"score":79.9,"version":"A.X"},{"company":"zai","family":"GLM","model":"glm-4.7","place":96,"score":79.9,"version":"GLM 4.7"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":97,"score":79.88,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":98,"score":79.88,"version":"Kanana 1.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":99,"score":79.87,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":100,"score":79.5,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":101,"score":79.3,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4","place":102,"score":79.3,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":103,"score":79.27,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":104,"score":79.2,"version":"Gemma 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":105,"score":79.2,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":106,"score":78.7,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b-chat","place":107,"score":78.7,"version":"CodeQwen1.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":108,"score":78.6,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":109,"score":78.43,"version":"DeepSeek R1"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":110,"score":78.33,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":111,"score":78.05,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":112,"score":78,"version":"Gemma 3"},{"company":"map","family":"OpenCoder","model":"opencoder-8b-instruct","place":113,"score":77.4,"version":"OpenCoder"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":114,"score":77.4,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":115,"score":76.83,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":116,"score":76.2,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":117,"score":75.68,"version":"Granite 3.3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":118,"score":75.61,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":119,"score":75.6,"version":"Gemini 1.5"},{"company":"skt","family":"A.X","model":"ax-k1","place":120,"score":75.6,"version":"A.X"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":121,"score":75.6,"version":"DeepSeek Coder V2"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":122,"score":75.6,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":123,"score":75.3,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":124,"score":75.26,"version":"Granite 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":125,"score":75,"version":"Falcon 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":126,"score":75,"version":"DeepSeek Coder"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":127,"score":75,"version":"Granite 4.0"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":128,"score":74.7,"version":"SmollM3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":129,"score":74,"version":"Granite 4.0"},{"company":"mistral","family":"Codestral","model":"codestral-22b-v0.1","place":130,"score":73.8,"version":"Codestral v0.1"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-33b","place":131,"score":73.8,"version":"OpenCodeInterpreter DS"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":132,"score":73.78,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":133,"score":73.78,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":134,"score":73.78,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":135,"score":73.39,"version":"Granite 3.2"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-33b-v1.1","place":136,"score":73.2,"version":"WizardCoder V1.1"},{"company":"artigenz","family":"Artigenz-Coder-DS-6.7B","model":"artigenz-coder-ds-6.7b","place":137,"score":72.6,"version":"Artigenz Coder DS"},{"company":"meta","family":"Llama3-70B-instruct","model":"llama3-70b-instruct","place":138,"score":72,"version":"Llama3-70B-instruct"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b-instruct-v0.1","place":139,"score":72,"version":"Mixtral v0.1"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-6.7b","place":140,"score":72,"version":"OpenCodeInterpreter DS"},{"company":"speechless","family":"speechless-codellama-34B-v2.0","model":"speechless-codellama-34b-v2.0","place":141,"score":72,"version":"speechless-codellama-34B-v2.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":142,"score":72,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":143,"score":71.95,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":144,"score":71.3,"version":"DeepSeek Coder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-7b-instruct-v1.5","place":145,"score":71.3,"version":"DeepSeek Coder"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":146,"score":71.3,"version":"Magicoder-S-DS-6.7B"},{"company":"huggingface","family":"starchat2-15b-v0.1","model":"starchat2-15b-v0.1","place":147,"score":71.3,"version":"starchat2-15b-v0.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":148,"score":70.9,"version":"Kimi Linear"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":149,"score":70.73,"version":"Falcon H1"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-nov-2023","place":150,"score":70.7,"version":"GPT-3.5"},{"company":"code","family":"code-millenials-34B","model":"code-millenials-34b","place":151,"score":70.7,"version":"code-millenials-34B"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":152,"score":70.7,"version":"Qwen3"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":153,"score":70.7,"version":"Ouro"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":154,"score":70.7,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":155,"score":70.6,"version":"Qwen2.5"},{"company":"databricks","family":"databricks/dbrx-instruct","model":"databricks-dbrx-instruct","place":156,"score":70.1,"version":"databricks/dbrx-instruct"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":157,"score":69.8,"version":"OLMo 3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":158,"score":69.51,"version":"LFM2"},{"company":"wavecoder","family":"WaveCoder-Ultra-6.7B","model":"wavecoder-ultra-6.7b","place":159,"score":69.5,"version":"WaveCoder-Ultra-6.7B"},{"company":"xwin","family":"XwinCoder-34B","model":"xwincoder-34b","place":160,"score":69.5,"version":"XwinCoder-34B"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":161,"score":69.1,"version":"Magpie"},{"company":"anthropic","family":"claude-3-haiku","model":"claude-3-haiku-mar-2024","place":162,"score":68.9,"version":"claude-3-haiku (Mar 2024)"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":163,"score":68.3,"version":"Codestral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":164,"score":68.29,"version":"Qwen2.5"},{"company":"magicoder","family":"Magicoder-S-CL-7B","model":"magicoder-s-cl-7b","place":165,"score":67.7,"version":"Magicoder-S-CL-7B"},{"company":"openchat","family":"OpenChat-3.5-7B-0106","model":"openchat-3.5-7b-0106","place":166,"score":67.7,"version":"OpenChat-3.5-7B-0106"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":167,"score":67.7,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":168,"score":67.5,"version":"Gemma 2"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":169,"score":67.4,"version":"Ouro"},{"company":"phind","family":"Phind-CodeLlama-34B-v2","model":"phind-codellama-34b-v2","place":170,"score":67.1,"version":"Phind-CodeLlama-34B-v2"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":171,"score":67,"version":"Gemma 2"},{"company":"openai","family":"GPT","model":"gpt-3.5-may-2023","place":172,"score":66.5,"version":"GPT-3.5"},{"company":"codellama","family":"CodeLlama","model":"codellama-70b-instruct","place":173,"score":65.9,"version":"CodeLlama"},{"company":"whiterabbit","family":"WhiteRabbitNeo-33B-v1","model":"whiterabbitneo-33b-v1","place":174,"score":65.9,"version":"WhiteRabbitNeo-33B-v1"},{"company":"speechless","family":"speechless-coder-ds-6.7B","model":"speechless-coder-ds-6.7b","place":175,"score":65.9,"version":"speechless-coder-ds-6.7B"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":176,"score":65.85,"version":"Falcon 3"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-python-34b-v1.0","place":177,"score":64.6,"version":"WizardCoder V1.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":178,"score":64.6,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"claude-3-sonnet","model":"claude-3-sonnet-mar-2024","place":179,"score":64,"version":"claude-3-sonnet (Mar 2024)"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":180,"score":62.91,"version":"DeepSeek R1"},{"company":"bigcode","family":"speechless-starcoder2-15b","model":"speechless-starcoder2-15b","place":181,"score":62.8,"version":"speechless-starcoder2-15b"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":182,"score":62.8,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":183,"score":62.2,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":184,"score":62.2,"version":"Qwen2.5"},{"company":"anthropic","family":"claude-2","model":"claude-2","place":185,"score":61.6,"version":"claude-2 (Mar 2024)"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":186,"score":61.59,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":187,"score":61.59,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":188,"score":61.59,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":189,"score":61,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":190,"score":60.98,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":191,"score":60.96,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-instruct","place":192,"score":60.4,"version":"DeepSeek Coder"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b-instruct-v0.1","place":193,"score":60.4,"version":"StarCoder2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":194,"score":60.37,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":195,"score":59.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":196,"score":59.8,"version":"Qwen2.5"},{"company":"code","family":"Code","model":"code-290k-6.7b-instruct","place":197,"score":59.7,"version":"Code 290k"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":198,"score":59.5,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-72b-chat","place":199,"score":59.1,"version":"Qwen1.5"},{"company":"microsoft","family":"Phi-3-mini-4k-instruct","model":"phi-3-mini-4k-instruct","place":200,"score":59.1,"version":"Phi-3-mini-4k-instruct"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":201,"score":57.93,"version":"LFM2"},{"company":"mistral","family":"dolphin-2.6-mixtral-8x7b","model":"dolphin-2.6-mixtral-8x7b","place":202,"score":57.3,"version":"dolphin-2.6-mixtral-8x7b"},{"company":"cohere","family":"Command","model":"command-r","place":203,"score":56.7,"version":"Command"},{"company":"meta","family":"Llama3-8B-instruct","model":"llama3-8b-instruct","place":204,"score":56.7,"version":"Llama3-8B-instruct"},{"company":"google","family":"Gemini","model":"gemini-1.0-pro","place":205,"score":55.5,"version":"Gemini 1.0"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":206,"score":53.7,"version":"Dream 7B"},{"company":"code","family":"Code-13B","model":"code-13b","place":207,"score":52.4,"version":"Code-13B"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":208,"score":51.83,"version":"Trinity Large"},{"company":"google","family":"Codegemma","model":"codegemma-7b-it","place":209,"score":51.8,"version":"Codegemma"},{"company":"bigcode","family":"speechless-starcoder2-7b","model":"speechless-starcoder2-7b","place":210,"score":51.8,"version":"speechless-starcoder2-7b"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":211,"score":50.61,"version":"Qwen2.5"},{"company":"meta","family":"CodeLlama","model":"codellama-70b","place":212,"score":50.6,"version":"CodeLlama"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-15b-v1.0","place":213,"score":50.6,"version":"WizardCoder V1.0"},{"company":"anthropic","family":"claude-instant-1","model":"claude-instant-1-mar-2024","place":214,"score":50.6,"version":"claude-instant-1 (Mar 2024)"},{"company":"speechless","family":"speechless-coding-7B-16k-tora","model":"speechless-coding-7b-16k-tora","place":215,"score":50.6,"version":"speechless-coding-7B-16k-tora"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":216,"score":50,"version":"Llama 3.2"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":217,"score":50,"version":"Dream"},{"company":"code","family":"Code-33B","model":"code-33b","place":218,"score":49.4,"version":"Code-33B"},{"company":"openhermes","family":"OpenHermes-2.5-Code-290k-13B","model":"openhermes-2.5-code-290k-13b","place":219,"score":48.8,"version":"OpenHermes-2.5-Code-290k-13B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":220,"score":48.78,"version":"Qwen2.5"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-instruct","place":221,"score":47.6,"version":"CoDA"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":222,"score":46.3,"version":"Qwen2.5"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":223,"score":46.3,"version":"Moonlight"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":224,"score":45.73,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b","place":225,"score":45.7,"version":"CodeQwen1.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":226,"score":45.12,"version":"Falcon H1"},{"company":"microsoft","family":"WizardCoder-Python-7B-V1.0","model":"wizardcoder-python-7b-v1.0","place":227,"score":45.1,"version":"WizardCoder-Python-7B-V1.0"},{"company":"microsoft","family":"phi-2-2.7B","model":"phi-2-2.7b","place":228,"score":45.1,"version":"phi-2-2.7B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-base","place":229,"score":44.5,"version":"DeepSeek Coder"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":230,"score":44.4,"version":"OLMo 2"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":231,"score":43.9,"version":"CodeLlama-34B"},{"company":"swissai","family":"Apertus","model":"apertus-70b-instruct","place":232,"score":42.9,"version":"Apertus"},{"company":"mistral","family":"Mistral","model":"mistral-codealpaca-7b","place":233,"score":42.1,"version":"Mistral Codealpaca"},{"company":"mistral","family":"MistralHermes","model":"mistralhermes-codepro-7b-v1","place":234,"score":42.1,"version":"MistralHermes CodePro v1"},{"company":"google","family":"Codegemma","model":"codegemma-7b","place":235,"score":41.5,"version":"codegemma-7b"},{"company":"speechless","family":"speechless-code-mistral-7B-v1.0","model":"speechless-code-mistral-7b-v1.0","place":236,"score":41.5,"version":"speechless-code-mistral-7B-v1.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-base","place":237,"score":39.6,"version":"DeepSeek Coder"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b-instruct-v0.1","place":238,"score":39.6,"version":"Mixtral v0.1"},{"company":"codellama","family":"CodeLlama-13B","model":"codellama-13b","place":239,"score":38.4,"version":"CodeLlama-13B"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b","place":240,"score":37.8,"version":"StarCoder2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":241,"score":37.2,"version":"Gemma 3"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":242,"score":37.2,"version":"Solar Mini"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":243,"score":37.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":244,"score":37.19,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":245,"score":36,"version":"Mistral v0.2"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":246,"score":35.4,"version":"CodeLlama-7B"},{"company":"google","family":"Gemma","model":"gemma-1.1-7b-it","place":247,"score":35.4,"version":"gemma-1.1"},{"company":"xdan","family":"xDAN-L1-Chat-RL-v1-7B","model":"xdan-l1-chat-rl-v1-7b","place":248,"score":32.9,"version":"xDAN-L1-Chat-RL-v1-7B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":249,"score":32.32,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":250,"score":31.7,"version":"LLaDA"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":251,"score":31.1,"version":"Llama 3.1"},{"company":"python","family":"Python-Code-13B","model":"python-code-13b","place":252,"score":30.5,"version":"Python-Code-13B"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-7b","place":253,"score":29.9,"version":"StarCoder2"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":254,"score":29.88,"version":"Llama 3.2"},{"company":"meta","family":"Llama3-8B-base","model":"llama3-8b-base","place":255,"score":29.3,"version":"Llama3-8B-base"},{"company":"bigcode","family":"StarCoder-15B","model":"starcoder-15b","place":256,"score":29.3,"version":"StarCoder"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":257,"score":29.3,"version":"Gemma 3"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":258,"score":28.7,"version":"gemma-7b"},{"company":"salesforce","family":"CodeGen","model":"codegen-16b","place":259,"score":28,"version":"CodeGen"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-3b","place":260,"score":27.4,"version":"StarCoder2"},{"company":"salesforce","family":"CodeT5+-16B","model":"codet5-16b","place":261,"score":26.8,"version":"CodeT5+-16B"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":262,"score":26.2,"version":"Llama 3.2"},{"company":"salesforce","family":"CodeGen","model":"codegen-6b","place":263,"score":25.6,"version":"CodeGen"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-base","place":264,"score":25.6,"version":"DeepSeek Coder"},{"company":"stable","family":"stable-code-3B","model":"stable-code-3b","place":265,"score":25.6,"version":"stable-code-3B"},{"company":"google","family":"Gemma","model":"gemma-7b-it","place":266,"score":25,"version":"Gemma"},{"company":"salesforce","family":"CodeT5+-6B","model":"codet5-6b","place":267,"score":24.4,"version":"CodeT5+-6B"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":268,"score":24.06,"version":"Llama 3.2"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-base","place":269,"score":23.8,"version":"CoDA"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":270,"score":23.8,"version":"Mistral"},{"company":"huggingface","family":"Zephyr","model":"zephyr-beta-7b","place":271,"score":23.2,"version":"Zephyr β-7B"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":272,"score":23.17,"version":"Llama 3.2"},{"company":"salesforce","family":"CodeGen","model":"codegen-2b","place":273,"score":22.6,"version":"CodeGen"},{"company":"salesforce","family":"CodeT5+-2B","model":"codet5-2b","place":274,"score":22,"version":"CodeT5+-2B"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-7b","place":275,"score":21.3,"version":"StarCoderBase"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":276,"score":20.73,"version":"Falcon 3"},{"company":"google","family":"Codegemma","model":"codegemma-2b","place":277,"score":20.7,"version":"codegemma-2b"},{"company":"google","family":"Gemma","model":"gemma-2b","place":278,"score":20.7,"version":"gemma-2b"},{"company":"salesforce","family":"CodeGen","model":"codegen2-7b","place":279,"score":17.7,"version":"CodeGen2"},{"company":"google","family":"gemma-1.1-2b-it","model":"gemma-1.1-2b-it","place":280,"score":17.7,"version":"gemma-1.1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":281,"score":17.4,"version":"Llama 3.2"},{"company":"salesforce","family":"CodeGen","model":"codegen2-16b","place":282,"score":16.5,"version":"CodeGen2"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-3b","place":283,"score":15.9,"version":"StarCoderBase"},{"company":"lmsys","family":"Vicuna-13B","model":"vicuna-13b","place":284,"score":15.9,"version":"Vicuna-13B"},{"company":"google","family":"Gemma","model":"gemma-2b-it","place":285,"score":15.2,"version":"Gemma"},{"company":"bigcode","family":"SantaCoder","model":"santacoder-1.1b","place":286,"score":14,"version":"SantaCoder"},{"company":"salesforce","family":"CodeGen","model":"codegen2-3b","place":287,"score":12.8,"version":"CodeGen2"},{"company":"ai21","family":"InCoder","model":"incoder-6.7b","place":288,"score":12.2,"version":"InCoder"},{"company":"bigcode","family":"StarCoder","model":"starcoderbase-1b","place":289,"score":12.2,"version":"StarCoderBase"},{"company":"lmsys","family":"Vicuna-7B","model":"vicuna-7b","place":290,"score":11.6,"version":"Vicuna-7B"},{"company":"openai","family":"GPT","model":"gpt-j-6b","place":291,"score":11,"version":"GPT-J"},{"company":"ai21","family":"InCoder","model":"incoder-1.3b","place":292,"score":11,"version":"InCoder"},{"company":"salesforce","family":"CodeGen","model":"codegen2-1b","place":293,"score":9.1,"version":"CodeGen2"},{"company":"openai","family":"GPT","model":"gpt-neo-2.7b","place":294,"score":6.7,"version":"GPT-Neo"},{"company":"cmu","family":"PolyCoder-2.7B","model":"polycoder-2.7b","place":295,"score":6.1,"version":"PolyCoder-2.7B"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":296,"score":5.5,"version":"Gemma 3"},{"company":"stabilityai","family":"StableLM-7B","model":"stablelm-7b","place":297,"score":2.4,"version":"StableLM-7B"},{"company":"zysecai","family":"Zyte","model":"zyte-1b","place":298,"score":1.8,"version":"Zyte"}],"stars":"1577","top_company":"anthropic","top_model":"Claude Sonnet 4","top_model_family":"Claude","top_model_id":"claude-sonnet-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":94.51,"topic":"Code generation","total_models":1613,"url":"https://github.com/evalplus/evalplus"},{"bench_models":5,"description":"HumanEval variant with visual programming prompts.","github":"","has_recent":0,"id":"humaneval-v","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HumanEval-V","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":1,"score":66.05,"version":"Step3-VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":2,"score":31.96,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":3,"score":29.26,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":4,"score":26.94,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":5,"score":24.31,"version":"InternVL 3.5"}],"stars":"","top_company":"stepfun","top_model":"Step3-VL-10B","top_model_family":"Step","top_model_id":"step3-vl-10b","top_model_version":"Step3-VL","top_open_source":1,"top_percent":66.05,"topic":"Code generation (vision)","total_models":1613,"url":""},{"bench_models":6,"description":"Multilingual code generation benchmark extending HumanEval to multiple programming languages.","github":"https://github.com/THUDM/CodeGeeX","has_recent":0,"id":"humaneval-x","leaderboard":"","lower_is_better":false,"metric":"percent","name":"HumanEval-X","paper":"","placements":[{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":1,"score":92.67,"version":"TeleChat3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":2,"score":89.76,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":3,"score":88.29,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":4,"score":87.3,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":5,"score":84.88,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":6,"score":76.1,"version":"Qwen3"}],"stars":"","top_company":"teleai","top_model":"TeleChat3-36B-Thinking","top_model_family":"TeleChat","top_model_id":"telechat3-36b-thinking","top_model_version":"TeleChat3","top_open_source":1,"top_percent":92.67,"topic":"Code generation (multilingual)","total_models":1613,"url":"https://github.com/THUDM/CodeGeeX"},{"bench_models":16,"description":"Hypersim benchmark for synthetic indoor scene understanding and reconstruction.","github":"","has_recent":0,"id":"hypersim","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Hypersim","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":1,"score":39.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":2,"score":39.3,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":3,"score":34.7,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":4,"score":14,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":5,"score":13.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":6,"score":13,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":7,"score":12.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":8,"score":12.7,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":9,"score":12.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":10,"score":12.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":11,"score":12.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":12,"score":12,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":13,"score":11,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":14,"score":11,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":15,"score":9.4,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":2.2,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 Mini Minimal","top_model_family":"GPT","top_model_id":"gpt-5-mini-minimal","top_model_version":"GPT-5","top_open_source":0,"top_percent":39.3,"topic":"3D scene understanding","total_models":1613,"url":""},{"bench_models":132,"description":"Instruction-following benchmark measuring compliance and adherence.","github":"","has_recent":0,"id":"ifbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"IFBench","paper":"","placements":[{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":1,"score":84.78,"version":"Mistral 3.2"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":2,"score":82.75,"version":"Mistral 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":76.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":76.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":76.1,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":6,"score":75.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":75.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":8,"score":75.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":9,"score":75,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5","place":10,"score":73.1,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":11,"score":73.1,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":12,"score":72,"version":"MiniMax M2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":13,"score":71.5,"version":"Nemotron 3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":14,"score":71.5,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":15,"score":71.47,"version":"GPT-5.2"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":16,"score":71.2,"version":"Intern-S1"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":17,"score":70.9,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":18,"score":70.8,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":19,"score":70.4,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":20,"score":70.2,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":21,"score":70.2,"version":"Qwen3.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":22,"score":70,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":23,"score":70,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":24,"score":69.7,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":25,"score":69.5,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":26,"score":69.4,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":27,"score":69,"version":"GPT-OSS"},{"company":"allenai","family":"OLMo","model":"olmo-3.1-think-32b","place":28,"score":68.1,"version":"OLMo 3.1"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":29,"score":67.3,"version":"K-EXAONE"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-a4b","place":30,"score":65,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":31,"score":65,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":32,"score":62.5,"version":"DeepSeek V3.2"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":33,"score":61.7,"version":"Apriel 1.5"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":34,"score":61.2,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":35,"score":61,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":36,"score":60.7,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":37,"score":60.7,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":38,"score":60.5,"version":"GPT-OSS"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":39,"score":60.45,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":40,"score":58.7,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":41,"score":58,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":42,"score":57.3,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":43,"score":57,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":44,"score":57,"version":"Claude 4.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":45,"score":55.8,"version":"Apriel 1.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":46,"score":55.4,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":47,"score":55.1,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":48,"score":54.7,"version":"Claude 4"},{"company":"xai","family":"Grok","model":"grok-4","place":49,"score":53.7,"version":"Grok 4"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":50,"score":53.7,"version":"Solar Open"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":51,"score":53.4,"version":"Falcon H1R"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":52,"score":52.6,"version":"Qwen3"},{"company":"ai21","family":"Jamba","model":"jamba-reasoning-3b","place":53,"score":52,"version":"Jamba Reasoning"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":54,"score":51.7,"version":"Phi-4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":55,"score":51.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":56,"score":51.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":57,"score":51,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":58,"score":50.5,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":59,"score":50.3,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":60,"score":50,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":61,"score":49.8,"version":"Qwen3"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":62,"score":49,"version":"Mi:dm K 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":63,"score":48.7,"version":"Gemini 2.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":64,"score":48.3,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking-2507","place":65,"score":48.06,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":66,"score":47.8,"version":"Kanana 2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":67,"score":47.6,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":68,"score":47.6,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":69,"score":47.33,"version":"LFM2.5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":70,"score":46.67,"version":"Ling 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":71,"score":45.6,"version":"GPT-5"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":72,"score":44.85,"version":"LFM2.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":73,"score":44.4,"version":"Magistral 1.2"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b-exp","place":74,"score":44.4,"version":"LFM2"},{"company":"skt","family":"A.X","model":"ax-k1","place":75,"score":44.3,"version":"A.X"},{"company":"zai","family":"GLM","model":"glm-4.5","place":76,"score":44.1,"version":"GLM 4.5"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":77,"score":43,"version":"Magistral 1.2"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":78,"score":43,"version":"Llama 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":79,"score":42.53,"version":"Kimi K2.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":80,"score":42.3,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":81,"score":41.84,"version":"Kanana 2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":82,"score":41.7,"version":"Kimi K2"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":83,"score":41.7,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":84,"score":41.4,"version":"Nemotron Cascade"},{"company":"menlo","family":"Jan","model":"jan-v1-2509","place":85,"score":40.9,"version":"Jan v1"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":86,"score":40.8,"version":"Nemotron Cascade"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":87,"score":39.6,"version":"DeepSeek R1"},{"company":"menlo","family":"Jan","model":"jan-v1","place":88,"score":39.1,"version":"Jan v1"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":89,"score":38.78,"version":"Kanana 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":90,"score":38,"version":"DeepSeek R1"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":91,"score":38,"version":"HyperCLOVA X"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":92,"score":37.8,"version":"DeepSeek V3.1"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":93,"score":37.8,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":94,"score":37.6,"version":"GLM 4.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":95,"score":37.4,"version":"Nemotron Nano v2"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":96,"score":37.3,"version":"Qwen3"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":97,"score":37,"version":"Llama Nemotron v1.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":98,"score":37,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":99,"score":36.7,"version":"Qwen2.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":100,"score":36.7,"version":"GLM 4.6"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":101,"score":36.7,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":102,"score":36.4,"version":"OLMo 2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":103,"score":36.3,"version":"EXAONE 4.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":104,"score":36.1,"version":"Gemini 2.5"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":105,"score":36,"version":"EXAONE 4.0"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":106,"score":36,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":107,"score":35.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":108,"score":35.3,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":109,"score":34.4,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":110,"score":34.4,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":111,"score":34.3,"version":"Nemotron-H"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":112,"score":33.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":113,"score":33,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":114,"score":31.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":115,"score":30.28,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":116,"score":30.27,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":117,"score":29.25,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":118,"score":29.2,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":119,"score":28,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":120,"score":27.8,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":121,"score":27,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":122,"score":26,"version":"Llama 3.2"},{"company":"swissai","family":"Apertus","model":"apertus-70b-instruct","place":123,"score":26,"version":"Apertus"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":124,"score":25.88,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":125,"score":25.85,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":126,"score":24.93,"version":"Granite 4.0"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":127,"score":24.73,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":128,"score":24,"version":"Granite 4.0"},{"company":"openai","family":"GPT","model":"gpt-4","place":129,"score":24,"version":"GPT-4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":130,"score":23.8,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-it","place":131,"score":23.73,"version":"Gemma 3n"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":132,"score":23.67,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":133,"score":23.53,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":134,"score":22.09,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":135,"score":21.33,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":136,"score":21.27,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-4-mini","place":137,"score":21,"version":"Phi 4"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":138,"score":21,"version":"Granite 4.0"},{"company":"liquidai","family":"LFM","model":"lfm2-1.2b","place":139,"score":20.7,"version":"LFM2"},{"company":"liquidai","family":"LFM","model":"lfm2-700m","place":140,"score":20.56,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":141,"score":20.47,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":142,"score":19.93,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":143,"score":19.75,"version":"Qwen3"},{"company":"menlo","family":"Jan","model":"jan-v1-edge","place":144,"score":18.51,"version":"Jan v1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":145,"score":18.37,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":146,"score":17.72,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":147,"score":16.86,"version":"Llama 3.2"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":148,"score":16.41,"version":"LFM2"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":149,"score":15.93,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":150,"score":15.93,"version":"Llama 3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":151,"score":13,"version":"DeepSeek R1"}],"stars":"70","top_company":"mistral","top_model":"Mistral Small 3.2 24B Instruct","top_model_family":"Mistral","top_model_id":"mistral-small-3.2-24b-instruct","top_model_version":"Mistral 3.2","top_open_source":1,"top_percent":84.78,"topic":"Instruction following","total_models":1613,"url":"https://github.com/allenai/IFBench"},{"bench_models":248,"description":"Instruction following capability evaluation for LLMs.","github":"https://github.com/google-research/google-research/tree/master/instruction_following_eval","has_recent":0,"id":"ifeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"IFEval","paper":"https://arxiv.org/abs/2311.07911","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":1,"score":95,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":94.8,"version":"GPT-5.2"},{"company":"openai","family":"o3","model":"o3-mini-high","place":3,"score":93.9,"version":"o3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":93.9,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":5,"score":93.9,"version":"GPT-5"},{"company":"allenai","family":"OLMo","model":"olmo-3.1-think-32b","place":6,"score":93.8,"version":"OLMo 3.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":93.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":8,"score":93.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":9,"score":93.4,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":10,"score":92.6,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":11,"score":92.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":12,"score":91.9,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":13,"score":91.4,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":14,"score":90.9,"version":"Claude 4.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":15,"score":90.4,"version":"Gemma 3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":16,"score":90.3,"version":"Nemotron Nano v2"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":17,"score":90.2,"version":"Gemma 3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":18,"score":90.2,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":19,"score":90,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":20,"score":89.94,"version":"Llama 3.3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":21,"score":89.87,"version":"Granite 4.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":22,"score":89.8,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":23,"score":89.8,"version":"Kimi K2"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":24,"score":89.7,"version":"K-EXAONE"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":25,"score":89.5,"version":"GPT-OSS"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":26,"score":89.4,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":27,"score":89.4,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":28,"score":89.37,"version":"Falcon H1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":29,"score":89,"version":"OLMo 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":30,"score":88.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":31,"score":88.9,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":32,"score":88.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":33,"score":88.9,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":34,"score":88.9,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":35,"score":88.7,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":36,"score":88.7,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":37,"score":88.6,"version":"Llama 3.1"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":38,"score":88.42,"version":"LFM2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":39,"score":88.4,"version":"Llama 3.1"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b-exp","place":40,"score":88.13,"version":"LFM2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":41,"score":88,"version":"ERNIE 4.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":42,"score":88,"version":"GPT-4.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":43,"score":88,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":44,"score":88,"version":"DeepSeek V3"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":45,"score":88,"version":"Solar Open"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":46,"score":87.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":47,"score":87.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":48,"score":87.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":49,"score":87.6,"version":"Qwen3 Next"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":50,"score":87.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":51,"score":87.6,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":52,"score":87.55,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":53,"score":87.5,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":54,"score":87.4,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":55,"score":87.4,"version":"Claude 4"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":56,"score":87.25,"version":"Kanana 2"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":57,"score":87.2,"version":"InternVL"},{"company":"zai","family":"GLM","model":"glm-4.6","place":58,"score":87.2,"version":"GLM 4.6"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":59,"score":86.97,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":60,"score":86.69,"version":"Nemotron Nano V2"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":61,"score":86.69,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":62,"score":86.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":63,"score":86.5,"version":"Qwen3"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":64,"score":86.5,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":65,"score":86.5,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":66,"score":86.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":67,"score":86.35,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":68,"score":86.32,"version":"Llama 4"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":69,"score":86.32,"version":"Falcon H1R"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":70,"score":86.3,"version":"openPangu-R"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":71,"score":86.23,"version":"LFM2.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":72,"score":86.1,"version":"Nemotron Nano v2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":73,"score":86.1,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":74,"score":86,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":75,"score":85.8,"version":"Qwen3 VL"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":76,"score":85.8,"version":"OLMo 2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":77,"score":85.62,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":78,"score":85.5,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":79,"score":85.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":80,"score":85.4,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":81,"score":85.4,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":82,"score":85.4,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":83,"score":85.35,"version":"Falcon H1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":84,"score":85.22,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":85,"score":85.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":86,"score":85.1,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":87,"score":85.05,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":88,"score":85,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":89,"score":85,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":90,"score":85,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":91,"score":84.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":92,"score":84.8,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":93,"score":84.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":94,"score":84.7,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":95,"score":84.7,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":96,"score":84.66,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":97,"score":84.6,"version":"Apriel Nemotron"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":98,"score":84.47,"version":"Kanana 2"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":99,"score":84.37,"version":"Solar Pro"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":100,"score":84.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":101,"score":84.29,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":102,"score":84.13,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":103,"score":84.1,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":104,"score":84.1,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":105,"score":84.1,"version":"Qwen3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":106,"score":84,"version":"Mi:dm 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":107,"score":83.93,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":108,"score":83.9,"version":"GPT-4o"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":109,"score":83.9,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":110,"score":83.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":111,"score":83.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":112,"score":83.7,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":113,"score":83.7,"version":"Nemotron Cascade"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":114,"score":83.7,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":115,"score":83.67,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":116,"score":83.6,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":117,"score":83.6,"version":"EXAONE 3.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":118,"score":83.5,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":119,"score":83.43,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":120,"score":83.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":121,"score":83.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":122,"score":83.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":123,"score":83.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":124,"score":83.2,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":125,"score":83.2,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":126,"score":83.18,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":127,"score":83.1,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":128,"score":83,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":129,"score":82.96,"version":"TeleChat3"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":130,"score":82.9,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":131,"score":82.81,"version":"Llama 3.3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":132,"score":82.8,"version":"QwQ"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":133,"score":82.72,"version":"Granite 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":134,"score":82.7,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":135,"score":82.62,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":136,"score":82.6,"version":"Tulu 3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":137,"score":82.5,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":138,"score":82.44,"version":"GLM 4.7"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":139,"score":82.4,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":140,"score":82.4,"version":"Tulu 3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":141,"score":82.4,"version":"GPT-OSS"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":142,"score":82.36,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":143,"score":82.31,"version":"Granite 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":144,"score":82.3,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":145,"score":82.3,"version":"Qwen3 VL"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":146,"score":82.2,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":147,"score":82.2,"version":"Kanana 2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":148,"score":82.1,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":149,"score":81.9,"version":"Qwen2.5"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":150,"score":81.9,"version":"Nemotron Cascade"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":151,"score":81.9,"version":"Hermes 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":152,"score":81.89,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":153,"score":81.89,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":154,"score":81.79,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":155,"score":81.75,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":156,"score":81.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":157,"score":81.7,"version":"Qwen3 VL"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":158,"score":81.7,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":159,"score":81.52,"version":"Ling 2.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":160,"score":81.38,"version":"Granite 4.0"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":161,"score":81.25,"version":"LFM2"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":162,"score":81.2,"version":"Youtu-LLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":163,"score":81.1,"version":"DeepSeek V3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":164,"score":81.1,"version":"Tulu 3"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b-instruct","place":165,"score":81.1,"version":"EXAONE 3.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":166,"score":81.06,"version":"Granite 4.0"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":167,"score":81.02,"version":"OLMo 2"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":168,"score":81,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-5b-instruct","place":169,"score":80.8,"version":"Apriel"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":170,"score":80.71,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":171,"score":80.66,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":172,"score":80.6,"version":"Llama 3.1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":173,"score":80.6,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-2-72b","place":174,"score":80.56,"version":"Qwen2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":175,"score":80.4,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":176,"score":80.2,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":177,"score":80.2,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":178,"score":80.08,"version":"Granite 4.0"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":179,"score":79.9,"version":"Llama 3.1 Nemotron"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":180,"score":79.9,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":181,"score":79.7,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":182,"score":79.61,"version":"Granite 4.0"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":183,"score":79.56,"version":"LFM2"},{"company":"openai","family":"o1","model":"o1-mini","place":184,"score":79.5,"version":"o1"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":185,"score":79.48,"version":"Kanana 1.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":186,"score":79.41,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":187,"score":79.12,"version":"Granite 4.0"},{"company":"arcee","family":"Llama Spark","model":"llama-spark-8b","place":188,"score":79.11,"version":"Llama Spark 8B"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":189,"score":79.11,"version":"Motif 2"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":190,"score":79.1,"version":"openPangu-R"},{"company":"openai","family":"GPT","model":"gpt-4","place":191,"score":79,"version":"GPT-4"},{"company":"aquif","family":"Aquif","model":"aquif-3-3.2b","place":192,"score":78.9,"version":"Aquif 3"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-2.7b","place":193,"score":78.9,"version":"Aquif 3.5"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":194,"score":78.84,"version":"Falcon 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":195,"score":78.7,"version":"DeepSeek R1"},{"company":"skt","family":"A.X","model":"ax-k1","place":196,"score":78.6,"version":"A.X"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":197,"score":78.45,"version":"Granite 4.0"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":198,"score":77.58,"version":"LFM2"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":199,"score":77.45,"version":"MiniCPM 4.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":200,"score":77.4,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":201,"score":77.04,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":202,"score":76.85,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":203,"score":76.8,"version":"Llama 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":204,"score":76.7,"version":"SmollM3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":205,"score":76.59,"version":"Falcon 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":206,"score":76.52,"version":"Motif 2"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":207,"score":76.34,"version":"MiniCPM SALA"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":208,"score":76.09,"version":"Granite 4.0"},{"company":"ai21","family":"Jamba","model":"jamba-mini-1.7","place":209,"score":76,"version":"Jamba Mini"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":210,"score":76,"version":"Hermes 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":211,"score":75.9,"version":"MiMo VL"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":212,"score":75.78,"version":"Motif 2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":213,"score":75.52,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":214,"score":75.36,"version":"Gemma 2"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":215,"score":75.3,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":216,"score":75.25,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":217,"score":75.1,"version":"Qwen2.5 VL"},{"company":"liquidai","family":"LFM","model":"lfm2-1.2b","place":218,"score":74.89,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":219,"score":74.82,"version":"Granite 3.3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":220,"score":74.7,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-it","place":221,"score":74.57,"version":"Gemma 3n"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":222,"score":74.55,"version":"SmollM3"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":223,"score":74.31,"version":"Granite 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":224,"score":74,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":225,"score":73.98,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":226,"score":73.68,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b","place":227,"score":73.6,"version":"EXAONE 3.5"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-mini-instruct","place":228,"score":73.6,"version":"Mi:dm 2.0"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":229,"score":73.49,"version":"Llama 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":230,"score":73.2,"version":"Granite 3.1"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":231,"score":73.08,"version":"Mixtral"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":232,"score":72.8,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":233,"score":72.5,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":234,"score":72.44,"version":"SmollM3"},{"company":"liquidai","family":"LFM","model":"lfm2-700m","place":235,"score":72.23,"version":"LFM2"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":236,"score":72.07,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":237,"score":71.65,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":238,"score":71.6,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":239,"score":71.43,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":240,"score":71.2,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":241,"score":71.2,"version":"SmolLM3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":242,"score":71,"version":"EXAONE Deep"},{"company":"swissai","family":"Apertus","model":"apertus-70b-instruct","place":243,"score":70.4,"version":"Apertus"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":244,"score":70.4,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":245,"score":70.06,"version":"Ministral 3"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":246,"score":69.9,"version":"Gemma 2"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":247,"score":69.8,"version":"Llama 3.1"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":248,"score":69.15,"version":"Phi 3.5 MoE"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":249,"score":69.1,"version":"Falcon 3"},{"company":"microsoft","family":"Phi","model":"phi-4-mini-3.8b","place":250,"score":68.6,"version":"Phi 4 mini"},{"company":"ai21","family":"Jamba","model":"jamba-mini-1.6","place":251,"score":68,"version":"Jamba Mini"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":252,"score":67.8,"version":"EXAONE 4.0"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":253,"score":67.22,"version":"EuroLLM"},{"company":"arcee","family":"AFM","model":"afm-4.5b","place":254,"score":67.03,"version":"AFM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":255,"score":66.5,"version":"DeepSeek R1"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":256,"score":65.88,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":257,"score":65.8,"version":"Granite 3.3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":258,"score":65.74,"version":"Mistral 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":259,"score":65.6,"version":"Qwen2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":260,"score":65.12,"version":"LFM2"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":261,"score":64.37,"version":"Phi 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":262,"score":64.24,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":263,"score":63.59,"version":"Granite 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":264,"score":63.25,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":265,"score":63.03,"version":"Granite 4.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":266,"score":62.5,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":267,"score":62.16,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":268,"score":62.1,"version":"Gemma 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":269,"score":62,"version":"MobileLLM"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":270,"score":61.55,"version":"Granite 3.2"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":271,"score":61.24,"version":"Apertus"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":272,"score":60.4,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":273,"score":59.5,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":274,"score":59.2,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":275,"score":59.1,"version":"DeepSeek R1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":276,"score":56.7,"version":"SmolLM2"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":277,"score":56.4,"version":"Ministral"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":278,"score":55.34,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":279,"score":54.26,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-1b","place":280,"score":53.5,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":281,"score":52.39,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":282,"score":52.37,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":283,"score":52.37,"version":"Llama 3.2"},{"company":"nvidia","family":"Nemotron Flash","model":"nemotron-flash-3b","place":284,"score":52.03,"version":"Nemotron Flash"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":285,"score":51.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":286,"score":48.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":287,"score":47.4,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":288,"score":46.64,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":289,"score":45.33,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":290,"score":40.3,"version":"Qwen2.5"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":291,"score":38.8,"version":"Magpie"},{"company":"loka","family":"TRLM","model":"trlm-135m","place":292,"score":35.49,"version":"TRLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":293,"score":34.6,"version":"DeepSeek R1"},{"company":"tii","family":"Falcon","model":"falcon-e-1b-base","place":294,"score":32.9,"version":"Falcon E"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":295,"score":32.9,"version":"Qwen2.5 Omni"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":296,"score":32.11,"version":"Qwen2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":297,"score":31.23,"version":"LFM2-Audio"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":298,"score":29.9,"version":"SmolLM2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":299,"score":29.4,"version":"DeepSeek R1"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":300,"score":28.48,"version":"LFM2.5-Audio"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":301,"score":26.7,"version":"K2 V2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":302,"score":24.4,"version":"SmolLM2"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":303,"score":24.28,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":304,"score":23.1,"version":"SmolLM1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":305,"score":21.15,"version":"SmolLM2"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":306,"score":17.4,"version":"K2 V2"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":307,"score":17.4,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":308,"score":15.1,"version":"Llama 3.0"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":309,"score":13.2,"version":"OLMo 3"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":310,"score":11.56,"version":"Mini-Omni2"},{"company":"kyutai","family":"Moshi","model":"moshi","place":311,"score":10.12,"version":"Moshi"}],"stars":"36312","top_company":"qwen","top_model":"Qwen3.5-27B","top_model_family":"Qwen","top_model_id":"qwen3.5-27b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":95,"topic":"Instruction following","total_models":1613,"url":"https://github.com/google-research/google-research/tree/master/ifeval"},{"bench_models":6,"description":"Instruction following evaluation for code generation tasks.","github":"","has_recent":0,"id":"ifeval-code","leaderboard":"","lower_is_better":false,"metric":"percent","name":"IFEval-Code","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":1,"score":28,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":2,"score":26,"version":"TeleChat3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":3,"score":25.73,"version":"GPT-OSS"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":4,"score":23,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":5,"score":20.95,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":6,"score":20.45,"version":"Qwen3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-32B","top_model_family":"Qwen","top_model_id":"qwen3-32b","top_model_version":"Qwen3","top_open_source":1,"top_percent":28,"topic":"Instruction following (code)","total_models":1613,"url":""},{"bench_models":10,"description":"IFEval strict prompt-level accuracy.","github":"","has_recent":0,"id":"ifeval-strict-prompt","leaderboard":"","lower_is_better":false,"metric":"percent","name":"IFEval (Strict Prompt)","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":1,"score":84.29,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":2,"score":83.73,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":3,"score":83.55,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":4,"score":83.36,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":5,"score":83.18,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":6,"score":82.62,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":7,"score":81.33,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":8,"score":81.15,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":9,"score":80.78,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":10,"score":76.16,"version":"Ling 2.0"}],"stars":"","top_company":"qwen","top_model":"Qwen3-8B Non-Thinking","top_model_family":"Qwen","top_model_id":"qwen3-8b-non-thinking","top_model_version":"Qwen3","top_open_source":1,"top_percent":84.29,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":22,"description":"Average of single-image visual question answering benchmarks.","github":"","has_recent":0,"id":"img-qa-avg","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Image QA Average","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":86.2,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":83.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":3,"score":81.9,"version":"GPT-5"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":4,"score":81.7,"version":"Molmo2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":81.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":6,"score":81.2,"version":"Qwen3 VL"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":7,"score":81.2,"version":"Eagle 2.5"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":8,"score":81.2,"version":"Molmo"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":9,"score":80.4,"version":"Molmo2"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":10,"score":79.8,"version":"Keye-VL 1.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":11,"score":79.7,"version":"Molmo2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":12,"score":79.3,"version":"Gemini 2.5"},{"company":"unknown","family":"PLM","model":"plm-8b","place":13,"score":78.7,"version":"PLM"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":14,"score":78.4,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":15,"score":78.2,"version":"InternVL 3.5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":16,"score":77.7,"version":"MiniCPM V 4.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":17,"score":77.3,"version":"Molmo"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":18,"score":77.2,"version":"InternVL 3.5"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":19,"score":77,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":20,"score":76.3,"version":"Claude 4.5"},{"company":"unknown","family":"PLM","model":"plm-3b","place":21,"score":75.9,"version":"PLM"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":22,"score":74.6,"version":"Molmo"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":86.2,"topic":"Image QA (aggregate)","total_models":1613,"url":""},{"bench_models":62,"description":"Evaluates free-form solutions to International Mathematical Olympiad problems using expert-style grading rubrics.","github":"https://github.com/google-deepmind/superhuman/tree/main/imobench","has_recent":0,"id":"imo-answerbench","leaderboard":"https://imobench.github.io/#leaderboard","lower_is_better":false,"metric":"percent","name":"IMO AnswerBench","paper":"https://arxiv.org/abs/2511.01846","placements":[{"company":"antgroup","family":"Ring","model":"ring-1t-2.5-heavy-thinking","place":1,"score":90,"version":"Ring 2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":2,"score":89.3,"version":"Seed 2.0"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":3,"score":86.8,"version":"LongCat-Flash"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":86.7,"version":"Gemini 3"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5","place":5,"score":86.69,"version":"Ring 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":6,"score":86.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":7,"score":86.3,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":8,"score":86.3,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":9,"score":86.3,"version":"GPT-5.2"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":10,"score":85.4,"version":"Step 3.5 Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":11,"score":84.5,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":12,"score":84.4,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":13,"score":84,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":14,"score":84,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":15,"score":83.9,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":83.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":17,"score":83.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":18,"score":82.8,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":19,"score":82,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":20,"score":81.8,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":21,"score":81.8,"version":"Kimi K2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":22,"score":81.6,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":23,"score":81.3,"version":"Gemini 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":24,"score":80.9,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":25,"score":80.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-deep-think-imo-gold","place":26,"score":80,"version":"Gemini Deep Think"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":27,"score":78.7,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":28,"score":78.6,"version":"Kimi K2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":29,"score":78.6,"version":"LongCat-Flash"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":30,"score":78.5,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":31,"score":78.3,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":32,"score":78.3,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":33,"score":77.7,"version":"DeepSeek V3.2"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":34,"score":77.3,"version":"Intern-S1"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":35,"score":76.3,"version":"Seed 1.8"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":36,"score":76.3,"version":"K-EXAONE"},{"company":"openai","family":"GPT","model":"gpt-5","place":37,"score":76,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":38,"score":76,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":39,"score":75.6,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":40,"score":74.8,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":41,"score":73.5,"version":"GLM 4.6"},{"company":"xai","family":"Grok","model":"grok-4","place":42,"score":73.1,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":43,"score":73,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":44,"score":72.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":45,"score":72.3,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":46,"score":72.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-deep-think","place":47,"score":71.8,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":48,"score":71.6,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":49,"score":68.2,"version":"Gemini 2.5"},{"company":"openai","family":"o4","model":"o4-mini-high","place":50,"score":67.9,"version":"o4"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":51,"score":66.1,"version":"EXAONE 4.0"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":52,"score":65.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":53,"score":65.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":54,"score":65.6,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":55,"score":62.31,"version":"Ling 2.5"},{"company":"openai","family":"o3","model":"o3","place":56,"score":61.1,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":57,"score":60.8,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":58,"score":60.7,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":59,"score":60.4,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":60,"score":57.5,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":61,"score":54.81,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":62,"score":54.34,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":63,"score":53.8,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":64,"score":53.4,"version":"Nanbeige4.1"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":65,"score":53.38,"version":"Nanbeige4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":66,"score":52.56,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":67,"score":48,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":68,"score":46.66,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":69,"score":45.8,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":70,"score":45.8,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":71,"score":43.94,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":72,"score":43.41,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":73,"score":41.81,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":74,"score":38.25,"version":"Nanbeige4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":75,"score":37,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":76,"score":36.56,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":77,"score":23,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":78,"score":22.3,"version":"Claude 4"}],"stars":"","top_company":"antgroup","top_model":"Ring-1T-2.5-heavy-thinking","top_model_family":"Ring","top_model_id":"ring-1t-2.5-heavy-thinking","top_model_version":"Ring 2.5","top_open_source":1,"top_percent":90,"topic":"Math (competition)","total_models":1613,"url":"https://imobench.github.io/"},{"bench_models":36,"description":"Evaluates inclusive language use and bias mitigation in model outputs.","github":"","has_recent":0,"id":"include","leaderboard":"","lower_is_better":false,"metric":"percent","name":"INCLUDE","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":90.5,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":87.5,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":86.2,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":85.6,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":5,"score":83.9,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":83.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":7,"score":82.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":8,"score":82.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":9,"score":81.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":10,"score":81.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":11,"score":81,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":12,"score":81,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":13,"score":79.7,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":14,"score":79.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":15,"score":78.9,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":16,"score":78.9,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":17,"score":77.2,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":18,"score":77.2,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":19,"score":75.3,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":20,"score":74.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":21,"score":74.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":22,"score":74,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":23,"score":73.7,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":24,"score":71.9,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":25,"score":71.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":26,"score":71.6,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":27,"score":71.4,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":28,"score":70.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":29,"score":67,"version":"Qwen3 VL"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":30,"score":63.97,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":31,"score":62.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":32,"score":61.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":33,"score":60.1,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":34,"score":51.62,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":35,"score":51.1,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":36,"score":51.06,"version":"Granite 4.0"},{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":37,"score":45.1,"version":"Aya"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":90.5,"topic":"Inclusiveness / bias","total_models":1613,"url":""},{"bench_models":46,"description":"Information retrieval question answering benchmark evaluating factual responses.","github":"","has_recent":0,"id":"infoqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"InfoQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":86.9,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":2,"score":84.5,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":3,"score":83.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":83,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":5,"score":82.2,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":6,"score":82,"version":"InternVL 2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":82,"version":"Gemini 2.5"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":8,"score":81.9,"version":"Molmo"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":9,"score":81,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":10,"score":80.9,"version":"Gemini 2.5"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":11,"score":80.4,"version":"Eagle 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":12,"score":80.3,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":13,"score":80.3,"version":"GLM 4.1V"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":14,"score":80.1,"version":"Molmo2"},{"company":"unknown","family":"PLM","model":"plm-8b","place":15,"score":80,"version":"PLM"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":16,"score":79.6,"version":"MolmoE"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":17,"score":79.2,"version":"GPT-4o"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":18,"score":79.1,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":19,"score":78.6,"version":"Molmo2"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":20,"score":78,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":21,"score":77.9,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":22,"score":76.5,"version":"Qwen2 VL"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":23,"score":75.3,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4v","place":24,"score":75.1,"version":"GPT-4V"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":25,"score":74.9,"version":"LLaVA OneVision"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":26,"score":74.9,"version":"Keye-VL 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":27,"score":74.8,"version":"InternVL 2"},{"company":"unknown","family":"PLM","model":"plm-3b","place":28,"score":74.6,"version":"PLM"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":29,"score":74.3,"version":"Claude 3.5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":30,"score":73.4,"version":"MiniCPM V 4.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":31,"score":72.6,"version":"Molmo"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":32,"score":70,"version":"Molmo"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":33,"score":68.8,"version":"LLaVA OneVision"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":34,"score":67.2,"version":"Llama 3.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":35,"score":65.9,"version":"Claude 4.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":36,"score":63.6,"version":"Llama 3.2"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":37,"score":56.1,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":38,"score":55.6,"version":"Claude 3"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":39,"score":53.9,"version":"MolmoE"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":40,"score":50.8,"version":"Pixtral"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":41,"score":46,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":42,"score":41.6,"version":"Cambrian 1"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":43,"score":36.6,"version":"Phi 3.5"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":44,"score":31.5,"version":"xGen-MM"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":45,"score":29.4,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":46,"score":25.8,"version":"LLaVA 1.5"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":47,"score":21.4,"version":"PaliGemma"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":86.9,"topic":"Information-seeking QA","total_models":1613,"url":""},{"bench_models":5,"description":"Information extraction benchmark for economically valuable fields.","github":"","has_recent":0,"id":"information-extraction","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Information Extraction","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":1,"score":46.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":46.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":46,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":44.6,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":5,"score":43.7,"version":"Seed 1.8"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.5","top_model_family":"Claude","top_model_id":"claude-sonnet-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":46.9,"topic":"Information extraction","total_models":1613,"url":""},{"bench_models":5,"description":"Information processing benchmark for economically valuable tasks.","github":"","has_recent":0,"id":"information-processing","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Information Processing","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":56.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":55,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":54.3,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":53.9,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":52.4,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":56.5,"topic":"Information processing","total_models":1613,"url":""},{"bench_models":65,"description":"Visual question answering over infographics requiring reading, counting, and reasoning.","github":"","has_recent":0,"id":"infovqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"InfoVQA","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":1,"score":92.6,"version":"Kimi K2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":2,"score":91.2,"version":"Seed 1.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":3,"score":91.2,"version":"Ovis 2.6"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":4,"score":89.8,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":89.5,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":6,"score":88,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":7,"score":87.3,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":8,"score":87.2,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":9,"score":87,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":10,"score":86,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":84.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":84.3,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":84.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":14,"score":84,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":15,"score":83.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":16,"score":82.1,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":17,"score":82,"version":"Qwen3 VL"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":18,"score":81.9,"version":"Manzano"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":19,"score":81.4,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":20,"score":81,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":21,"score":80.3,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":22,"score":79.8,"version":"InternVL 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":23,"score":79,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":24,"score":78,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":25,"score":78,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":26,"score":77.1,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":27,"score":76.9,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":28,"score":76.12,"version":"Qwen2.5 VL"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":29,"score":75.8,"version":"FastVLM"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":30,"score":75,"version":"Manzano"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":31,"score":75,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":32,"score":73,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":33,"score":72.4,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":34,"score":72.1,"version":"InternVL 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":35,"score":71.9,"version":"Qwen3 VL"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":36,"score":71.6,"version":"jina-VLM"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":37,"score":70.8,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":38,"score":70.7,"version":"InternVL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":39,"score":70.6,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":40,"score":69.4,"version":"Claude 4.1"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":41,"score":69.3,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":42,"score":69.29,"version":"InternVL 3.5"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":43,"score":67.37,"version":"LFM2-VL"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":44,"score":67.3,"version":"MM1.5"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":45,"score":67.1,"version":"InternVL 3"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":46,"score":66.1,"version":"InternVL 3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":47,"score":65.5,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":48,"score":64,"version":"Qwen2 VL"},{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":49,"score":62.71,"version":"LFM2.5-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":50,"score":60.99,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":51,"score":60.9,"version":"InternVL 2.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":52,"score":59.7,"version":"FastVLM"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":53,"score":58.5,"version":"MM1.5"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":54,"score":58.35,"version":"LFM2-VL"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":55,"score":57.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":56,"score":54.5,"version":"Gemini Nano"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":57,"score":51,"version":"Bagel"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":58,"score":49.2,"version":"GPT-5"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":59,"score":49,"version":"Phi-3"},{"company":"apple","family":"MM1","model":"mm1-30b","place":60,"score":47.3,"version":"MM1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":61,"score":46.9,"version":"Qwen2.5 Omni"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":62,"score":46.4,"version":"FastVLM"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":63,"score":37.75,"version":"SmolVLM2"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":64,"score":37.6,"version":"MiniCPM V 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":65,"score":30,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":66,"score":30,"version":"Claude 4"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":67,"score":23.92,"version":"FastVLM"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":68,"score":21.3,"version":"Janus Pro"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":69,"score":14.1,"version":"Janus Pro"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":70,"score":0,"version":"VILA 1.5"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":71,"score":0,"version":"Cambrian"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":72,"score":0,"version":"BLIP-3O"},{"company":"baai","family":"Emu","model":"emu3-8b","place":73,"score":0,"version":"Emu3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":74,"score":0,"version":"GPT-4o"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2.5 Thinking","top_model_family":"Kimi","top_model_id":"kimi-k2.5-thinking","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":92.6,"topic":"Infographic VQA","total_models":1613,"url":""},{"bench_models":5,"description":"Intent recognition benchmark for practical applications.","github":"","has_recent":0,"id":"intention-recognition","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Intention Recognition","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":65.3,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":63.8,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":63.2,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":62.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":58.3,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":65.3,"topic":"Intent recognition","total_models":1613,"url":""},{"bench_models":15,"description":"Intuitive physics reasoning benchmark.","github":"","has_recent":0,"id":"intphys-2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"IntPhys 2","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-flash","place":1,"score":63.4,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":2,"score":59.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":58.3,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":4,"score":58,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":56.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":56.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":56.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":56,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":9,"score":55,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":10,"score":54.9,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":11,"score":54.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":12,"score":53,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":13,"score":52,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":14,"score":51.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":15,"score":51,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Flash","top_model_family":"Gemini","top_model_id":"gemini-3-flash","top_model_version":"Gemini 3","top_open_source":0,"top_percent":63.4,"topic":"Intuitive physics","total_models":1613,"url":""},{"bench_models":12,"description":"Inverse instruction-following evaluation.","github":"","has_recent":0,"id":"inverse-ifeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Inverse IFEval","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-flash","place":1,"score":80.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":80.6,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":80.3,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":79.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":78.9,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":6,"score":78.9,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":7,"score":77.1,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":75.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":9,"score":74.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":72.4,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":11,"score":72.3,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":70.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":69.3,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":14,"score":69.3,"version":"Seed 2.0"}],"stars":"","top_company":"google","top_model":"Gemini 3 Flash","top_model_family":"Gemini","top_model_id":"gemini-3-flash","top_model_version":"Gemini 3","top_open_source":0,"top_percent":80.9,"topic":"Instruction following (inverse)","total_models":1613,"url":""},{"bench_models":3,"description":"Relative throughput on ISL/OSL 8k/16k context workloads.","github":"","has_recent":0,"id":"isl-osl-8k-16k","leaderboard":"","lower_is_better":false,"metric":"relative_throughput","name":"ISL/OSL 8k/16k","paper":"","placements":[{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":1,"score":3.3,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-a4b","place":2,"score":1.5,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":3,"score":1,"version":"Qwen3"}],"stars":"","top_company":"nvidia","top_model":"Nemotron-3-Nano-30B-A3B","top_model_family":"Nemotron","top_model_id":"nemotron-3-nano-30b-a3b","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":3.3,"topic":"Throughput","total_models":1613,"url":""},{"bench_models":10,"description":"A benchmark measuring LLM judging ability.","github":"","has_recent":0,"id":"judgemark-v2-1","leaderboard":"https://eqbench.com/judgemark-v2.html","lower_is_better":false,"metric":"percent","name":"JudgeMark v2.1","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":1,"score":81.99,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":81.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":3,"score":76.31,"version":"GPT-4.1"},{"company":"openai","family":"o3","model":"o3","place":4,"score":75.1,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":5,"score":74.49,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":6,"score":73.09,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":72.06,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":8,"score":63.41,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":9,"score":62.97,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":10,"score":61.08,"version":"GPT-5"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4","top_model_family":"Claude","top_model_id":"claude-sonnet-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":81.99,"topic":"LLM judging ability","total_models":1613,"url":""},{"bench_models":5,"description":"Korean safety benchmark evaluating harmfulness and compliance.","github":"","has_recent":0,"id":"kgc-safety","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KGC-Safety","paper":"","placements":[{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":1,"score":96.1,"version":"K-EXAONE"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":2,"score":92.5,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":3,"score":73,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":4,"score":66.2,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":5,"score":58,"version":"EXAONE 4.0"}],"stars":"","top_company":"lg","top_model":"K-EXAONE","top_model_family":"K-EXAONE","top_model_id":"k-exaone","top_model_version":"K-EXAONE","top_open_source":1,"top_percent":96.1,"topic":"Safety (Korean)","total_models":1613,"url":""},{"bench_models":5,"description":"Keep/kill working-memory benchmark with 4 people entities.","github":"","has_recent":0,"id":"kk-4-people","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KK-4 People","paper":"","placements":[{"company":"mbzuai","family":"K2","model":"k2-v2","place":1,"score":92.9,"version":"K2 V2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":2,"score":42.4,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":3,"score":26.1,"version":"Qwen2.5"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":4,"score":7.6,"version":"Llama 3.1"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":5,"score":4.8,"version":"K2 V2"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":6,"score":4.2,"version":"Llama 3.0"}],"stars":"","top_company":"mbzuai","top_model":"K2-V2","top_model_family":"K2","top_model_id":"k2-v2","top_model_version":"K2 V2","top_open_source":1,"top_percent":92.9,"topic":"Working memory (4 people)","total_models":1613,"url":""},{"bench_models":5,"description":"Keep/kill working-memory benchmark with 8 people entities.","github":"","has_recent":0,"id":"kk-8-people","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KK-8 People","paper":"","placements":[{"company":"mbzuai","family":"K2","model":"k2-v2","place":1,"score":82.8,"version":"K2 V2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":2,"score":13,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":3,"score":5.7,"version":"Qwen2.5"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":4,"score":1.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":5,"score":1.1,"version":"Llama 3.0"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":6,"score":0.5,"version":"K2 V2"}],"stars":"","top_company":"mbzuai","top_model":"K2-V2","top_model_family":"K2","top_model_id":"k2-v2","top_model_version":"K2 V2","top_open_source":1,"top_percent":82.8,"topic":"Working memory (8 people)","total_models":1613,"url":""},{"bench_models":11,"description":"Korean Massive Multitask Language Understanding benchmark.","github":"","has_recent":0,"id":"kmmlu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KMMLU","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":1,"score":78.7,"version":"DeepSeek V3.1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":2,"score":77.7,"version":"GLM 4.6"},{"company":"skt","family":"A.X","model":"ax-k1","place":3,"score":73,"version":"A.X"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":4,"score":68.26,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":5,"score":67.56,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":6,"score":67.32,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":7,"score":65.75,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":8,"score":63.49,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":9,"score":62.25,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":10,"score":62.15,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":11,"score":61.98,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":12,"score":61.56,"version":"Kanana 1.5"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.1","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.1","top_model_version":"DeepSeek V3.1","top_open_source":1,"top_percent":78.7,"topic":"Korean knowledge","total_models":1613,"url":"https://huggingface.co/datasets/HAERAE-HUB/KMMLU"},{"bench_models":57,"description":"Korean Multilingual Massive Multitask Language Understanding Pro","github":"","has_recent":0,"id":"kmmlu-pro","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KMMLU-Pro","paper":"","placements":[{"company":"openai","family":"o1","model":"o1","place":1,"score":77.54,"version":"o1"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":2,"score":75.98,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":3,"score":75.51,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":4,"score":73.33,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":5,"score":72.81,"version":"GPT-4.1"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":6,"score":72.73,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":7,"score":72.35,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":8,"score":72.1,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":9,"score":71.6,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":10,"score":70.54,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":11,"score":69.42,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":12,"score":68.2,"version":"GLM 4.6"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":13,"score":68.03,"version":"Grok 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":14,"score":67.9,"version":"DeepSeek V3.1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":15,"score":67.7,"version":"EXAONE 4.0"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":16,"score":67.3,"version":"K-EXAONE"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":17,"score":66.45,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":18,"score":66.03,"version":"EXAONE 4.0"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta","place":19,"score":65.89,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":20,"score":64.75,"version":"Qwen3"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":21,"score":64.55,"version":"QwQ"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":22,"score":64.32,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":23,"score":64.01,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":24,"score":63.86,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":25,"score":63.01,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":26,"score":62.4,"version":"GPT-OSS"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":27,"score":62.24,"version":"EXAONE 4.0"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":28,"score":62.06,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":29,"score":61.97,"version":"Qwen3"},{"company":"cohere","family":"Command","model":"command-a","place":30,"score":61.46,"version":"Command"},{"company":"skt","family":"A.X","model":"ax-k1","place":31,"score":60.3,"version":"A.X"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini-2025-04-14","place":32,"score":60.15,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":33,"score":58.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":34,"score":56.39,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":35,"score":55.27,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":36,"score":54.63,"version":"Llama 3.3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":37,"score":53.02,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":38,"score":50.91,"version":"Mistral 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":39,"score":46.92,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-4-14b","place":40,"score":44.33,"version":"Phi 4"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b-thinking","place":41,"score":42.69,"version":"EXAONE 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":42,"score":41.06,"version":"Gemma 3"},{"company":"onelineai","family":"Ko-R1","model":"ko-r1-7b-v2.1","place":43,"score":38.7,"version":"Ko-R1"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":44,"score":38.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":45,"score":38.27,"version":"Qwen3"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-text-instruct-1.5b","place":46,"score":38.1,"version":"HyperCLOVA X"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":47,"score":37.48,"version":"EXAONE 4.0"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":48,"score":33.81,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-3b-instruct","place":49,"score":33.27,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":50,"score":32.86,"version":"Gemma 3"},{"company":"kakao","family":"Kanana Nano","model":"kanana-nano-2.1b-instruct","place":51,"score":32.6,"version":"Kanana Nano"},{"company":"cohere","family":"Aya","model":"aya-expanse-8b","place":52,"score":31.65,"version":"Aya Expanse"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":53,"score":30.5,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":54,"score":30.42,"version":"Qwen3"},{"company":"cohere","family":"Aya","model":"aya-expanse-32b","place":55,"score":28.46,"version":"Aya Expanse"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":56,"score":25.53,"version":"Llama 3.2"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":57,"score":24.6,"version":"EXAONE Deep"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":58,"score":21.6,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":59,"score":20.55,"version":"DeepSeek R1"}],"stars":"","top_company":"openai","top_model":"o1","top_model_family":"o1","top_model_id":"o1","top_model_version":"o1","top_open_source":0,"top_percent":77.54,"topic":"Multilingual knowledge","total_models":1613,"url":"https://huggingface.co/datasets/HAERAE-HUB/KMMLU"},{"bench_models":54,"description":"Redux variant of KMMLU benchmark","github":"","has_recent":0,"id":"kmmlu-redux","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KMMLU-Redux","paper":"https://arxiv.org/abs/2507.08924","placements":[{"company":"openai","family":"o1","model":"o1","place":1,"score":81.14,"version":"o1"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":2,"score":79.92,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":3,"score":79.36,"version":"Claude 3.7"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":4,"score":78.51,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":5,"score":77.58,"version":"Llama 4"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":6,"score":76.88,"version":"Claude 3.7"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":7,"score":75.9,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":8,"score":75.86,"version":"GPT-4.1"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":9,"score":75.8,"version":"o4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":10,"score":74.49,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":11,"score":73.4,"version":"GLM 4.6"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":12,"score":72.9,"version":"Grok 3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":13,"score":72.71,"version":"EXAONE 4.0"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta","place":14,"score":71.47,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":15,"score":69.54,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":16,"score":68.77,"version":"Qwen3"},{"company":"skt","family":"A.X","model":"ax-k1","place":17,"score":68.3,"version":"A.X"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":18,"score":67.84,"version":"o3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":19,"score":67.49,"version":"Llama 4"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":20,"score":67.34,"version":"QwQ"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini-2025-04-14","place":21,"score":67.03,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":22,"score":65.71,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":23,"score":65.64,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":24,"score":65.25,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":25,"score":64.98,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":26,"score":64.79,"version":"EXAONE 4.0"},{"company":"cohere","family":"Command","model":"command-a","place":27,"score":62.93,"version":"Command"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":28,"score":58.79,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":29,"score":58.41,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":30,"score":57.25,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":31,"score":56.17,"version":"Llama 3.3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":32,"score":54.04,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":33,"score":52.92,"version":"Mistral 3.1"},{"company":"microsoft","family":"Phi","model":"phi-4-14b","place":34,"score":49.75,"version":"Phi 4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":35,"score":49.25,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b-thinking","place":36,"score":46.85,"version":"EXAONE 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":37,"score":46.7,"version":"Gemma 3"},{"company":"onelineai","family":"Ko-R1","model":"ko-r1-7b-v2.1","place":38,"score":41.94,"version":"Ko-R1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":39,"score":40.43,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":40,"score":38,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":41,"score":37.8,"version":"Qwen3"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-text-instruct-1.5b","place":42,"score":33.94,"version":"HyperCLOVA X"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":43,"score":33.7,"version":"SmolLM3"},{"company":"cohere","family":"Aya","model":"aya-expanse-32b","place":44,"score":33.05,"version":"Aya Expanse"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":45,"score":31.89,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":46,"score":28.99,"version":"Qwen3"},{"company":"cohere","family":"Aya","model":"aya-expanse-8b","place":47,"score":28.3,"version":"Aya Expanse"},{"company":"kakao","family":"Kanana Nano","model":"kanana-nano-2.1b-instruct","place":48,"score":27.25,"version":"Kanana Nano"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":49,"score":25.09,"version":"Gemma 3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":50,"score":25,"version":"EXAONE Deep"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-3b-instruct","place":51,"score":24.74,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":52,"score":24.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":53,"score":21.3,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":54,"score":17.59,"version":"Llama 3.2"}],"stars":"","top_company":"openai","top_model":"o1","top_model_family":"o1","top_model_id":"o1","top_model_version":"o1","top_open_source":0,"top_percent":81.14,"topic":"Multilingual knowledge","total_models":1613,"url":"https://huggingface.co/datasets/HAERAE-HUB/KMMLU"},{"bench_models":5,"description":"Long-context understanding benchmark in Korean.","github":"","has_recent":0,"id":"ko-longbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ko-LongBench","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":1,"score":87.9,"version":"DeepSeek V3.2"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":2,"score":86.8,"version":"K-EXAONE"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":3,"score":83.2,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":4,"score":82.2,"version":"GPT-OSS"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":5,"score":68,"version":"EXAONE 4.0"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.2-Thinking","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.2-thinking","top_model_version":"DeepSeek V3.2","top_open_source":1,"top_percent":87.9,"topic":"Korean long-context","total_models":1613,"url":""},{"bench_models":5,"description":"Korean benchmark for knowledge and language understanding.","github":"","has_recent":0,"id":"kobolt","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KoBALT","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":1,"score":62.7,"version":"DeepSeek V3.2"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":2,"score":61.8,"version":"K-EXAONE"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":3,"score":56.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":4,"score":54.3,"version":"GPT-OSS"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":5,"score":25.4,"version":"EXAONE 4.0"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.2-Thinking","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.2-thinking","top_model_version":"DeepSeek V3.2","top_open_source":1,"top_percent":62.7,"topic":"Korean knowledge","total_models":1613,"url":""},{"bench_models":5,"description":"Korean multi-turn chat evaluation benchmark.","github":"","has_recent":0,"id":"komt-bench","leaderboard":"","lower_is_better":false,"metric":"score","name":"KoMT-Bench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":1,"score":8.49,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":2,"score":8.24,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":3,"score":8.21,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":4,"score":7.94,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":5,"score":7.89,"version":"Qwen3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Instruct-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-instruct-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":8.49,"topic":"Korean chat ability","total_models":1613,"url":""},{"bench_models":20,"description":"Comprehensive reasoning benchmark spanning diverse domains and cognitive skills.","github":"","has_recent":0,"id":"kor-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KOR-Bench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":77.4,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":76.2,"version":"Seed 1.8"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":3,"score":76,"version":"Ling 1T"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":75,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":74.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":74.2,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":7,"score":73.76,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":8,"score":73.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":9,"score":70.56,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":10,"score":69.44,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":11,"score":69.2,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":12,"score":68.96,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":13,"score":68,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":14,"score":65.12,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":15,"score":64.24,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":16,"score":63.2,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":17,"score":63.04,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":18,"score":62.8,"version":"LLaDA 2.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":19,"score":59.68,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":20,"score":54.96,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":21,"score":49.92,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":22,"score":48,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":23,"score":46.64,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":24,"score":37.26,"version":"LLaDA 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":77.4,"topic":"Reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"Korean reasoning benchmark evaluating diverse reasoning capabilities.","github":"","has_recent":0,"id":"korbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KORBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":79.2,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":2,"score":77.5,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":77.4,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":4,"score":77,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":5,"score":76,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":6,"score":74.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":73.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":73,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":72.8,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":79.2,"topic":"General reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Korean simple question answering benchmark.","github":"","has_recent":0,"id":"kosimpleqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KoSimpleQA","paper":"","placements":[{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":1,"score":49.7,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":2,"score":49.4,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":3,"score":45.7,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":4,"score":26.33,"version":"Qwen3"}],"stars":"","top_company":"kakao","top_model":"Kanana-2-30B-A3B-Mid-2601","top_model_family":"Kanana","top_model_id":"kanana-2-30b-a3b-mid-2601","top_model_version":"Kanana 2","top_open_source":1,"top_percent":49.7,"topic":"Korean QA","total_models":1613,"url":""},{"bench_models":5,"description":"Korean STEM and math benchmark","github":"","has_recent":0,"id":"ksm","leaderboard":"","lower_is_better":false,"metric":"percent","name":"KSM","paper":"","placements":[{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":1,"score":60.9,"version":"EXAONE Deep"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":2,"score":60.6,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":3,"score":52.9,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":4,"score":49.7,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":5,"score":22.8,"version":"Qwen3"}],"stars":"","top_company":"lg","top_model":"EXAONE Deep 2.4B","top_model_family":"EXAONE","top_model_id":"exaone-deep-2.4b","top_model_version":"EXAONE Deep","top_open_source":1,"top_percent":60.9,"topic":"Multilingual math","total_models":1613,"url":""},{"bench_models":16,"description":"Word prediction requiring broad context understanding.","github":"","has_recent":0,"id":"lambada","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LAMBADA","paper":"https://arxiv.org/pdf/1606.06031","placements":[{"company":"openai","family":"GPT","model":"gpt-3","place":1,"score":86.4,"version":"GPT-3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":2,"score":78.3,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":3,"score":78.1,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":4,"score":77.6,"version":"Qwen2.5"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":5,"score":77.4,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":6,"score":77.2,"version":"Marin 32B"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":7,"score":72.9,"version":"Nemotron Nano v2"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":8,"score":71,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":9,"score":68.9,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":10,"score":65.1,"version":"Pythia"},{"company":"openai","family":"GPT","model":"gpt-2","place":11,"score":63.24,"version":"GPT-2"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":12,"score":61.9,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":13,"score":58,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":14,"score":52.4,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":15,"score":34.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":16,"score":19.2,"version":"Pythia"}],"stars":"","top_company":"openai","top_model":"GPT-3","top_model_family":"GPT","top_model_id":"gpt-3","top_model_version":"GPT-3","top_open_source":0,"top_percent":86.4,"topic":"Language modeling","total_models":1613,"url":"https://zenodo.org/record/2630551"},{"bench_models":4,"description":"Robustness to latent jailbreak adversarial techniques.","github":"https://github.com/qiuhuachuan/latent-jailbreak","has_recent":0,"id":"latent-jailbreak","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LatentJailbreak","paper":"https://arxiv.org/pdf/2307.08487.pdf","placements":[{"company":"openai","family":"GPT","model":"gpt-3.5-turbo","place":1,"score":77.4,"version":"GPT-3.5"},{"company":"belle","family":"BELLE","model":"belle-7b-2m","place":2,"score":59.6,"version":"BELLE 7B"},{"company":"mbzuai","family":"K2","model":"k2-think","place":3,"score":37,"version":"K2-THINK"},{"company":"zai","family":"ChatGLM","model":"chatglm2-6b","place":4,"score":24.7,"version":"ChatGLM2"}],"stars":"39","top_company":"openai","top_model":"GPT-3.5-turbo","top_model_family":"GPT","top_model_id":"gpt-3.5-turbo","top_model_version":"GPT-3.5","top_open_source":0,"top_percent":77.4,"topic":"Safety / jailbreak","total_models":1613,"url":""},{"bench_models":11,"description":"Vision-language QA benchmark v1.","github":"","has_recent":0,"id":"lbv1-qa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LBV1-QA","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":73.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":71.28,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":3,"score":70.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking-preview","place":4,"score":70.71,"version":"Qwen3"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1.5-30b-a3b","place":5,"score":70.4,"version":"QwenLong"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":6,"score":69.9,"version":"DeepSeek R1"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1","place":7,"score":68.4,"version":"QwenLong"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":8,"score":67.1,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":9,"score":67.1,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":10,"score":66.86,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":11,"score":66.3,"version":"GPT-OSS"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":73.7,"topic":"Vision-language","total_models":1613,"url":""},{"bench_models":11,"description":"Vision-language benchmark v2.","github":"","has_recent":0,"id":"lbv2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LBV2","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":65.72,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":62.82,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":3,"score":59.48,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking-preview","place":4,"score":57.89,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":5,"score":57.46,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":6,"score":56.77,"version":"Gemini 2.5"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1.5-30b-a3b","place":7,"score":55.27,"version":"QwenLong"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":8,"score":49.11,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":9,"score":47.01,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":10,"score":43.74,"version":"GPT-5"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1","place":11,"score":40.76,"version":"QwenLong"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":65.72,"topic":"Vision-language","total_models":1613,"url":""},{"bench_models":5,"description":"Length-based instruction-following evaluation benchmark.","github":"","has_recent":0,"id":"lifebench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LIFEBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":61.7,"version":"GPT-5.2"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":2,"score":57.9,"version":"Ling 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":3,"score":55.5,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":54.9,"version":"Kimi K2.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":5,"score":42.3,"version":"Ling 1T"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":61.7,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":11,"description":"Question answering benchmark for autonomous driving scene understanding.","github":"","has_recent":0,"id":"lingoqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LingoQA","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":1,"score":82,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":2,"score":81.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":80.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":4,"score":79.2,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":78.8,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":72.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":68.8,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":8,"score":68.2,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":9,"score":66.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":10,"score":62.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":12.8,"version":"Claude 4.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-27B","top_model_family":"Qwen","top_model_id":"qwen3.5-27b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":82,"topic":"Driving scene QA","total_models":1613,"url":""},{"bench_models":140,"description":"Continually updated capability benchmark across diverse tasks.","github":"","has_recent":0,"id":"livebench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LiveBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":82.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":2,"score":79.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":78.59,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":4,"score":78.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":5,"score":77.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":6,"score":77.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":7,"score":76.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":8,"score":76.6,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-5-medium","place":9,"score":76.45,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":10,"score":76.4,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":11,"score":76.4,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":12,"score":75.8,"version":"Qwen3 Next"},{"company":"openai","family":"o1","model":"o1","place":13,"score":75.7,"version":"o1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":14,"score":75.7,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":15,"score":75.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-low","place":16,"score":75.34,"version":"GPT-5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":17,"score":75.2,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":18,"score":74.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":19,"score":74.9,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":20,"score":74.8,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-pro-high","place":21,"score":74.72,"version":"o3"},{"company":"openai","family":"o3","model":"o3-high","place":22,"score":74.61,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":23,"score":74.6,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":24,"score":74.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":25,"score":74.3,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4.1-opus-thinking","place":26,"score":73.48,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4-opus-thinking","place":27,"score":72.93,"version":"Claude 4"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":28,"score":72.9,"version":"JoyAI-LLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":29,"score":72.4,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":30,"score":72.2,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4","place":31,"score":72.11,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":32,"score":72.1,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":33,"score":72.08,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-medium","place":34,"score":71.98,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":35,"score":71.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":36,"score":71.6,"version":"DeepSeek R1"},{"company":"openai","family":"o4","model":"o4-mini-high","place":37,"score":71.52,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":38,"score":71.3,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":39,"score":71.3,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-max-thinking","place":40,"score":70.95,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking-2507","place":41,"score":70.76,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":42,"score":70.75,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":43,"score":70.69,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":44,"score":70.3,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-2025-05-28","place":45,"score":70.1,"version":"DeepSeek R1"},{"company":"openai","family":"o3","model":"o3-mini","place":46,"score":70,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":47,"score":69.8,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":48,"score":69.39,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":49,"score":69,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":50,"score":68.9,"version":"ERNIE 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":51,"score":67.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":52,"score":67.7,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":53,"score":67.6,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":54,"score":67.43,"version":"Claude 3.7"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":55,"score":67.3,"version":"openPangu-R"},{"company":"openai","family":"o4","model":"o4-mini-medium","place":56,"score":66.87,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":57,"score":66.3,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":58,"score":65.93,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":59,"score":65.4,"version":"Qwen3 VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":60,"score":65.15,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":61,"score":64.93,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":62,"score":64.72,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":63,"score":64.42,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-low","place":64,"score":63.85,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":65,"score":63.71,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":66,"score":63.55,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":67,"score":63.37,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":68,"score":63,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":69,"score":62.7,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":70,"score":62.55,"version":"DeepSeek V3.1"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta-high","place":71,"score":62.36,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":72,"score":62,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":73,"score":60.9,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-chat","place":74,"score":60.78,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen-3-coder-480b-a35b-instruct","place":75,"score":60.45,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":76,"score":59.93,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":77,"score":59.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":78,"score":59.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":79,"score":59.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":80,"score":59.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-30b-a3b","place":81,"score":59.02,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":82,"score":58.74,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4.5-preview","place":83,"score":58.65,"version":"GPT-4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":84,"score":58.48,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-5-nano-high","place":85,"score":58.47,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-code-fast","place":86,"score":57.94,"version":"Grok Code"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite-preview-thinking","place":87,"score":57.53,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":88,"score":56.46,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":89,"score":56.19,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":90,"score":56.05,"version":"Grok 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":91,"score":55.99,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":92,"score":55.9,"version":"GPT-4.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":93,"score":55.41,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"chatgpt-4o","place":94,"score":54.74,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":95,"score":54.6,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":96,"score":54.21,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":97,"score":54.03,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":98,"score":53.5,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":99,"score":53.11,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":100,"score":52.92,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":101,"score":51.93,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-max","place":102,"score":51.93,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":103,"score":51.8,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":104,"score":51.57,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":105,"score":51.4,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":106,"score":51.34,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":107,"score":50.65,"version":"Mistral 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":108,"score":49.9,"version":"Motif 2"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":109,"score":49.27,"version":"Phi-4"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":110,"score":49.23,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-70b","place":111,"score":48.53,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":112,"score":47.78,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":113,"score":47.43,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-lite","place":114,"score":46.78,"version":"Gemini 2.0"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":115,"score":46.26,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":116,"score":45.74,"version":"Falcon H1"},{"company":"openai","family":"GPT","model":"gpt-5-nano-low","place":117,"score":45.45,"version":"GPT-5"},{"company":"cohere","family":"Command","model":"command-a","place":118,"score":44.17,"version":"Command"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-turbos","place":119,"score":43.99,"version":"Hunyuan Turbos"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":120,"score":43.31,"version":"Mistral"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":121,"score":43.1,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct-turbo","place":122,"score":42.51,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":123,"score":42,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":124,"score":41.44,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":125,"score":40.73,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-small","place":126,"score":40.55,"version":"Mistral"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":127,"score":40.51,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":128,"score":39.51,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":129,"score":37.13,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":130,"score":37.08,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":131,"score":36.86,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":132,"score":36.83,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":133,"score":36.7,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":134,"score":34.3,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":135,"score":34.13,"version":"Falcon H1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":136,"score":33.8,"version":"Motif 2"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":137,"score":32.35,"version":"Falcon 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":138,"score":32.09,"version":"GPT-5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":139,"score":31.73,"version":"Llama 3.1"},{"company":"cohere","family":"Command","model":"command-r-plus","place":140,"score":30.14,"version":"Command"},{"company":"google","family":"Gemma","model":"gemma-3n-e4b-it","place":141,"score":27.85,"version":"Gemma 3n"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":142,"score":27.78,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":143,"score":27.32,"version":"Qwen2.5"},{"company":"cohere","family":"Command","model":"command-r","place":144,"score":27.15,"version":"Command"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":145,"score":26.37,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":146,"score":26.01,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":147,"score":23.33,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-it","place":148,"score":21.74,"version":"Gemma 3n"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":149,"score":21.65,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":150,"score":20.8,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":151,"score":18.79,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":152,"score":14.97,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":153,"score":14.27,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":154,"score":14.1,"version":"Falcon 3"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":82.4,"topic":"General capability","total_models":1613,"url":""},{"bench_models":369,"description":"Live coding and execution-based evaluation benchmark (v6 dataset).","github":"","has_recent":0,"id":"livecodebench","leaderboard":"https://livecodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"LiveCodeBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":92,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":2,"score":91.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":3,"score":90.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":90.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":5,"score":89.9,"version":"GPT-OSS"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5-heavy-thinking","place":6,"score":89.4,"version":"Ring 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":7,"score":89,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-speciale","place":8,"score":88.7,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":88.1,"version":"Gemini 3"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5","place":10,"score":87.89,"version":"Ring 2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":11,"score":87.8,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":12,"score":87.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":13,"score":87.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":14,"score":87.7,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":87.4,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":16,"score":87.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":17,"score":87,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":18,"score":87,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":19,"score":87,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":20,"score":87,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":21,"score":87,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":22,"score":86.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":23,"score":86.6,"version":"GPT-5"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":24,"score":86.4,"version":"Step 3.5 Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":25,"score":86,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":26,"score":85.9,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":27,"score":85,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":28,"score":85,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":29,"score":85,"version":"Kimi K2.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":30,"score":84.9,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":31,"score":84.8,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":32,"score":84.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-codex","place":33,"score":84.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":34,"score":84.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":35,"score":84.5,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":36,"score":83.9,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":37,"score":83.6,"version":"Qwen3.5"},{"company":"xai","family":"Grok","model":"grok-4","place":38,"score":83.3,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":39,"score":83.3,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":40,"score":83.3,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":41,"score":83.2,"version":"GPT-OSS"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":42,"score":83.2,"version":"Grok 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":43,"score":83.1,"version":"Kimi K2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":44,"score":83,"version":"MiniMax M2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":45,"score":82.8,"version":"GLM 4.6"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":46,"score":82.8,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":47,"score":82.8,"version":"Claude 4.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":48,"score":82.8,"version":"LongCat-Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":49,"score":82.71,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":50,"score":82.7,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":51,"score":82.4,"version":"DeepSeek V3.2"},{"company":"openai","family":"o4","model":"o4-mini","place":52,"score":82.2,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":53,"score":82.2,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":54,"score":81.9,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":55,"score":81.9,"version":"GPT-OSS"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":56,"score":81.7,"version":"Seed 2.0"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-thinking","place":57,"score":81.1,"version":"IQuest Coder V1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":58,"score":81,"version":"MiniMax M2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":59,"score":80.8,"version":"DeepSeek V3.2"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":60,"score":80.7,"version":"K-EXAONE"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":61,"score":80.7,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":62,"score":80.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking-high","place":63,"score":80.6,"version":"GPT-5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":64,"score":80.6,"version":"MiMo V2"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":65,"score":80.5,"version":"IQuest Coder V1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":66,"score":80.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":67,"score":80.4,"version":"GPT-OSS"},{"company":"openai","family":"o4","model":"o4-mini-high","place":68,"score":80.2,"version":"o4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":69,"score":80.1,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":70,"score":80,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":71,"score":79.8,"version":"DeepSeek V3.1"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":72,"score":79.5,"version":"Seed 1.8"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":73,"score":79.4,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":74,"score":79.2,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":75,"score":79.2,"version":"MiniMax M2"},{"company":"xai","family":"Grok","model":"grok-4-fast-thinking","place":76,"score":79,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":77,"score":79,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":78,"score":78.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":79,"score":78.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":80,"score":78.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":81,"score":78.4,"version":"Qwen3 Next"},{"company":"antgroup","family":"Ring","model":"ring-1t-preview","place":82,"score":78.3,"version":"Ring"},{"company":"antgroup","family":"Ring","model":"ring-1t","place":83,"score":78.3,"version":"Ring"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":84,"score":77.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":85,"score":77.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":86,"score":77.1,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":87,"score":77,"version":"DeepSeek R1"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":88,"score":77,"version":"Llama Nemotron v1.5"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":89,"score":76.9,"version":"Nanbeige4.1"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":90,"score":76.43,"version":"Step3-VL"},{"company":"xai","family":"Grok","model":"grok-3-mini-fast-high-reasoning","place":91,"score":76.2,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":92,"score":76.2,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3-high","place":93,"score":75.8,"version":"o3"},{"company":"openai","family":"o3","model":"o3","place":94,"score":75.8,"version":"o3"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":95,"score":75.77,"version":"Step3-VL"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":96,"score":75.72,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":97,"score":75.33,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":98,"score":75.1,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":99,"score":75.1,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":100,"score":75,"version":"Magistral 1.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":101,"score":74.9,"version":"DeepSeek V3.1"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":102,"score":74.9,"version":"Magistral 1.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":103,"score":74.8,"version":"DeepSeek V3.1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":104,"score":74.7,"version":"EXAONE 4.0"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":105,"score":74.6,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":106,"score":74.6,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":107,"score":74.53,"version":"GPT-OSS"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":108,"score":74.3,"version":"Intern-S1"},{"company":"openai","family":"o4","model":"o4-mini-medium","place":109,"score":74.2,"version":"o4"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":110,"score":74.2,"version":"Solar Open"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":111,"score":74.1,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":112,"score":74.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":113,"score":74.01,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5","place":114,"score":73.8,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-06-05","place":115,"score":73.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":116,"score":73.6,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":117,"score":73.4,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":118,"score":73.3,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":119,"score":73.22,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":120,"score":73.1,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":121,"score":73,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":122,"score":73,"version":"Claude 4.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":123,"score":72.94,"version":"dots.vlm1"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":124,"score":72.8,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":125,"score":72.8,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":126,"score":72.3,"version":"Magistral 1.2"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":127,"score":72.1,"version":"Magistral 1.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":128,"score":72.01,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":129,"score":72,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":130,"score":72,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":131,"score":71.97,"version":"Qwen3 Next"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":132,"score":71.9,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-05-06","place":133,"score":71.8,"version":"Gemini 2.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":134,"score":71.6,"version":"Apriel 1.5"},{"company":"openai","family":"o3","model":"o3-mini","place":135,"score":71.5,"version":"o3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":136,"score":71.4,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":137,"score":71.1,"version":"Nemotron Nano v2"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":138,"score":71.1,"version":"Nemotron Cascade"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":139,"score":71,"version":"Claude 4.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":140,"score":70.88,"version":"Magistral 1.2"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":141,"score":70.76,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":142,"score":70.7,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":143,"score":70.65,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-3-beta-think","place":144,"score":70.6,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":145,"score":70.6,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":146,"score":70.5,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":147,"score":70.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":148,"score":70.2,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-thinking","place":149,"score":70.2,"version":"Claude 4"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":150,"score":70,"version":"EXAONE 4.0"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":151,"score":69.9,"version":"GPT-OSS"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":152,"score":69.82,"version":"Ring 2.0"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":153,"score":69.8,"version":"Nemotron OpenReasoning"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":154,"score":69.5,"version":"Gemini 2.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":155,"score":69.5,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":156,"score":69.45,"version":"Qwen3 VL"},{"company":"primeintellect","family":"INTELLECT","model":"intellect-3","place":157,"score":69.3,"version":"INTELLECT"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":158,"score":69.01,"version":"Seed OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":159,"score":69,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":160,"score":69,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":161,"score":68.7,"version":"Qwen3 Next"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":162,"score":68.6,"version":"Falcon H1R"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":163,"score":68.4,"version":"GLM 4.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":164,"score":68.3,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":165,"score":68.3,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":166,"score":68.2,"version":"Nemotron 3"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":167,"score":68.17,"version":"Ling 2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":168,"score":67.9,"version":"OLMo 3"},{"company":"nousresearch","family":"NousCoder","model":"nouscoder-14b","place":169,"score":67.87,"version":"NousCoder"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":170,"score":67.8,"version":"Nemotron OpenReasoning"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":171,"score":67.8,"version":"GLM 4.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":172,"score":67.6,"version":"OLMo 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":173,"score":67.51,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":174,"score":67.5,"version":"Seed 1.5"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-high","place":175,"score":67.4,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":176,"score":67.3,"version":"GLM 4.5"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":177,"score":67,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":178,"score":66.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":179,"score":66.89,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"opencodereasoning-nemotron-1.1-32b","place":180,"score":66.8,"version":"Nemotron OpenCodeReasoning"},{"company":"xai","family":"Grok","model":"grok-3-mini-high","place":181,"score":66.7,"version":"Grok 3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":182,"score":66.7,"version":"EXAONE 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":183,"score":66.59,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805-thinking","place":184,"score":66.5,"version":"Claude 4.1"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":185,"score":66.5,"version":"TeleChat3"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":186,"score":66.45,"version":"GPT-OSS"},{"company":"openai","family":"o3","model":"o3-mini","place":187,"score":66.3,"version":"o3"},{"company":"xai","family":"Grok","model":"grok-3-mini-fast-low-reasoning","place":188,"score":66.3,"version":"Grok 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":189,"score":66,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":190,"score":66,"version":"Qwen3"},{"company":"openai","family":"o4","model":"o4-mini-low","place":191,"score":65.9,"version":"o4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":192,"score":65.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":193,"score":65.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":194,"score":65.7,"version":"Qwen3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":195,"score":65.6,"version":"JoyAI-LLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":196,"score":65.5,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":197,"score":65.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":198,"score":65.4,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":199,"score":65.3,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":200,"score":65.3,"version":"Nemotron Nano v2"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":201,"score":65.22,"version":"QwQ"},{"company":"xbai","family":"XBai o4","model":"xbai-o4-medium","place":202,"score":65,"version":"XBai o4"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":203,"score":65,"version":"MiniMax M1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":204,"score":64.6,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":205,"score":64.5,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":206,"score":64.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":207,"score":64.1,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":208,"score":64.1,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":209,"score":64,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":210,"score":64,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":211,"score":64,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":212,"score":64,"version":"GLM 4.7"},{"company":"mbzuai","family":"K2","model":"k2-think","place":213,"score":63.97,"version":"K2-THINK"},{"company":"openai","family":"o1","model":"o1","place":214,"score":63.9,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":215,"score":63.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":216,"score":63.5,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":217,"score":63.3,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":218,"score":63.11,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-med","place":219,"score":63,"version":"o3"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":220,"score":63,"version":"HyperCLOVA X"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":221,"score":62.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":222,"score":62.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":223,"score":62.6,"version":"GPT-5"},{"company":"antgroup","family":"Ring","model":"ring-mini-2.0","place":224,"score":62.56,"version":"Ring 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":225,"score":62.5,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":226,"score":62.4,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":227,"score":62.33,"version":"Qwen3"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":228,"score":62.3,"version":"MiniMax M1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":229,"score":62.3,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":230,"score":62.28,"version":"Ling 1T"},{"company":"antgroup","family":"Ring","model":"ring-mini-sparse-2.0-exp","place":231,"score":62.19,"version":"Ring 2.0"},{"company":"xai","family":"Grok","model":"grok-code-fast","place":232,"score":62,"version":"Grok Code"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-05-20","place":233,"score":61.9,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":234,"score":61.9,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":235,"score":61.68,"version":"Ling 1T"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":236,"score":61.66,"version":"Motif 2"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":237,"score":61.5,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":238,"score":61.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":239,"score":61.2,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":240,"score":61.01,"version":"DeepSeek R1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":241,"score":61,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-a4b","place":242,"score":61,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":243,"score":61,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":244,"score":61,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":245,"score":60.8,"version":"Kanana 2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-04-17","place":246,"score":60.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":247,"score":60.6,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":248,"score":60.4,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":249,"score":60.1,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":250,"score":60,"version":"Nemotron Nano V2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":251,"score":59.7,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":252,"score":59.5,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.0","place":253,"score":59.36,"version":"Magistral 1.0"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.1","place":254,"score":59.35,"version":"Magistral 1.1"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.1","place":255,"score":59.17,"version":"Magistral 1.1"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":256,"score":58.8,"version":"Kanana 2"},{"company":"nvidia","family":"Nemotron","model":"llama-3.3-nemotron-super-thinking","place":257,"score":58.4,"version":"Nemotron Super"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":258,"score":58.3,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":259,"score":58.24,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":260,"score":58.2,"version":"GPT-4.1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":261,"score":57.79,"version":"Nemotron"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":262,"score":57.71,"version":"Falcon H1R"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":263,"score":57.71,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":264,"score":57.5,"version":"Qwen3"},{"company":"aquif","family":"Aquif","model":"aquif-3.6-1b","place":265,"score":57.5,"version":"Aquif 3.6"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":266,"score":57.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":267,"score":57.2,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":268,"score":57.2,"version":"DeepSeek R1"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":269,"score":57.1,"version":"Seed 1.5"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-low","place":270,"score":57,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-thinking","place":271,"score":56.94,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":272,"score":56.9,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":273,"score":56.7,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-thinking","place":274,"score":56.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":275,"score":56.6,"version":"Qwen3 Next"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":276,"score":56.6,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":277,"score":56.1,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":278,"score":55.9,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":279,"score":55.9,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":280,"score":55.9,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.0","place":281,"score":55.84,"version":"Magistral 1.0"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":282,"score":55.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":283,"score":55.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-medium","place":284,"score":54.9,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":285,"score":54.7,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":286,"score":54.6,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":287,"score":53.9,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":288,"score":53.71,"version":"Ministral 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":289,"score":53.7,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":290,"score":53.7,"version":"Kimi K2"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":291,"score":53.1,"version":"Phi-4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":292,"score":53,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":293,"score":53,"version":"Apriel 1.5"},{"company":"xai","family":"Grok","model":"grok-3","place":294,"score":52.9,"version":"Grok 3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":295,"score":52.48,"version":"Ling 2.0"},{"company":"nvidia","family":"AceReason","model":"acereason-nemotron-1.1","place":296,"score":52.1,"version":"1.1"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":297,"score":52,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":298,"score":51.8,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":299,"score":51.43,"version":"MiniCPM 4.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":300,"score":51.38,"version":"Ling 2.0"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":301,"score":51.3,"version":"K2 V2"},{"company":"weibo","family":"VibeThinker","model":"vibethinker-1.5b","place":302,"score":51.1,"version":"VibeThinker"},{"company":"openai","family":"o1","model":"o1","place":303,"score":50.3,"version":"o1"},{"company":"mistral","family":"Magistral","model":"magistral-medium-2506","place":304,"score":50.3,"version":"Magistral 2506"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":305,"score":50.03,"version":"Motif 2"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":306,"score":49.71,"version":"Falcon H1"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":307,"score":49.6,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":308,"score":49.4,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-7b-rl","place":309,"score":49.3,"version":"MiMo 7B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":310,"score":49.12,"version":"Qwen2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":311,"score":48.95,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":312,"score":48.9,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":313,"score":48.9,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":314,"score":48.72,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":315,"score":48.71,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-5","place":316,"score":48.57,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":317,"score":48.57,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":318,"score":48.5,"version":"Claude 4"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":319,"score":48.5,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":320,"score":48.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-thinking","place":321,"score":48.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":322,"score":48.02,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl-thinking","place":323,"score":47.5,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":324,"score":47.4,"version":"Claude 4"},{"company":"mistral","family":"Magistral","model":"magistral-small-2506","place":325,"score":47.4,"version":"Magistral 2506"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":326,"score":47.4,"version":"Nemotron-H"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":327,"score":47.3,"version":"Llama 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":328,"score":47.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":329,"score":46.9,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20-thinking","place":330,"score":46.9,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":331,"score":46.9,"version":"DeepSeek V3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":332,"score":46.9,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":333,"score":46.42,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":334,"score":46.1,"version":"Grok 4"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":335,"score":46,"version":"Nanbeige4"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":336,"score":45.7,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":337,"score":45.6,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":338,"score":45.43,"version":"Gemini 2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":339,"score":45.37,"version":"LLaDA 2.1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":340,"score":45.3,"version":"EXAONE 4.0"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3.1","place":341,"score":44.8,"version":"Mistral 3.1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":342,"score":44.7,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":343,"score":44.7,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":344,"score":44.11,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":345,"score":44.05,"version":"LLaDA 2.1"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":346,"score":43.7,"version":"Youtu-LLM"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":347,"score":43.6,"version":"Gemini 2.0"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":348,"score":43.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":349,"score":43.4,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":350,"score":43.4,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":351,"score":43.2,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":352,"score":43.1,"version":"EXAONE Deep"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":353,"score":42.7,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":354,"score":42.7,"version":"GPT-4.1"},{"company":"KunLun","family":"Skywork OR1","model":"skywork-or1-7b","place":355,"score":42.7,"version":"Skywork OR1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":356,"score":42.6,"version":"Qwen3 VL"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":357,"score":42.51,"version":"LLaDA 2.0"},{"company":"zai","family":"GLM","model":"glm-z1","place":358,"score":42.3,"version":"GLM Z1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":359,"score":42.29,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":360,"score":42.29,"version":"Ling 2.0"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku-20241022","place":361,"score":41.9,"version":"Claude 3.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":362,"score":41.9,"version":"openPangu-R"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":363,"score":41.7,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":364,"score":41.63,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":365,"score":40.5,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":366,"score":40,"version":"Kimi Dev"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":367,"score":39.91,"version":"JoyAI-LLM"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":368,"score":39.9,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":369,"score":39.71,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":370,"score":39.7,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":371,"score":39.3,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":372,"score":39,"version":"Gemma 3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":373,"score":38.8,"version":"ERNIE 4.5"},{"company":"xai","family":"Grok","model":"grok-2","place":374,"score":38.7,"version":"Grok 2"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":375,"score":38.5,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":376,"score":37.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":377,"score":37.34,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-large-2411","place":378,"score":37.1,"version":"Mistral Large"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":379,"score":37,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":380,"score":36.9,"version":"Gemini 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":381,"score":36.8,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":382,"score":36.7,"version":"Qwen2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":383,"score":36.4,"version":"Claude 3.5"},{"company":"meta","family":"Llama","model":"llama-3.3-instruct-turbo-70b","place":384,"score":36.3,"version":"Llama 3.3"},{"company":"nvidia","family":"Nemotron","model":"llama-3.3-nemotron-super-4","place":385,"score":36.3,"version":"Nemotron Super"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":386,"score":36.11,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":387,"score":36,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":388,"score":36,"version":"Qwen3"},{"company":"cohere","family":"Command","model":"command-a","place":389,"score":35.1,"version":"Command"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":390,"score":35.1,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":391,"score":35.03,"version":"Falcon H1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":392,"score":34.8,"version":"Ling 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":393,"score":34.5,"version":"Gemini 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-large-3-base","place":394,"score":34.4,"version":"Mistral 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":395,"score":34.2,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":396,"score":33.3,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":397,"score":32.8,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":398,"score":32.8,"version":"Llama 4"},{"company":"tencent","family":"HY","model":"hy-1.8b-2bit","place":399,"score":32.73,"version":"HY"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":400,"score":32.68,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":401,"score":32.6,"version":"GPT-4.1"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":402,"score":32.6,"version":"KAT"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":403,"score":32,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":404,"score":31.9,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":405,"score":31.83,"version":"LLaDA 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":406,"score":31.8,"version":"Mistral 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":407,"score":31.5,"version":"Qwen3"},{"company":"tencent","family":"HY","model":"hy-1.8b","place":408,"score":31.5,"version":"HY"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":409,"score":31.3,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":410,"score":30.92,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":411,"score":30.8,"version":"SmollM3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":412,"score":30.8,"version":"MiMo V2"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":413,"score":30.7,"version":"Seed OSS"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":414,"score":30.7,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":415,"score":30.7,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":416,"score":30.7,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":417,"score":30.4,"version":"LLaDA 2.1"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":418,"score":30.3,"version":"Mistral 3"},{"company":"tencent","family":"HY","model":"hy-1.8b-int4gptq","place":419,"score":30.08,"version":"HY"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":420,"score":29.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":421,"score":29.8,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":422,"score":29.7,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":423,"score":29.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":424,"score":29.1,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":425,"score":29.1,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":426,"score":29,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":427,"score":28.85,"version":"LLaDA 2.1"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":428,"score":28.7,"version":"GPT-4"},{"company":"aquif","family":"Aquif","model":"aquif-3-moe-17b-a2.8b","place":429,"score":28.6,"version":"Aquif 3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":430,"score":28.6,"version":"Nemotron OpenReasoning"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":431,"score":28.58,"version":"LLaDA 2.0"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":432,"score":28.1,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":433,"score":28.1,"version":"Kimi K2"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":434,"score":27.7,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-saferl","place":435,"score":27.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":436,"score":27.5,"version":"GPT-4o"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":437,"score":27.5,"version":"Mistral 3.1"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":438,"score":27.43,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":439,"score":27.4,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":440,"score":27.2,"version":"DeepSeek V3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":441,"score":26.9,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":442,"score":26.81,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":443,"score":26.76,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":444,"score":26.73,"version":"Granite 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":445,"score":26.6,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":446,"score":26.4,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":447,"score":26.4,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":448,"score":26.3,"version":"Kimi K2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-21b-a3b-pt","place":449,"score":26.1,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-nothinking-2504","place":450,"score":26.1,"version":"Qwen3"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-80b-a13b-instruct","place":451,"score":25.8,"version":"Hunyuan"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":452,"score":25.5,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-4-14b","place":453,"score":25.2,"version":"Phi 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":454,"score":25.1,"version":"Llama 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":455,"score":24.9,"version":"DeepSeek V3.2"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-nano-1b","place":456,"score":24.8,"version":"Aquif 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":457,"score":24.8,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":458,"score":24.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":459,"score":24.6,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":460,"score":24.07,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":461,"score":23.87,"version":"Falcon H1"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-2.7b","place":462,"score":23.1,"version":"Aquif 3.5"},{"company":"ai21","family":"Jamba","model":"jamba-1.6-large","place":463,"score":22.3,"version":"Jamba 1.6"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":464,"score":22.3,"version":"Seed Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":465,"score":21.3,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":466,"score":21.04,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":467,"score":20.4,"version":"Gemma 2"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":468,"score":20.2,"version":"Claude 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":469,"score":20,"version":"Kimi Linear"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":470,"score":19.9,"version":"DeepSeek R1"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":471,"score":19.9,"version":"MobileLLM R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":472,"score":19.8,"version":"DeepSeek R1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":473,"score":19.77,"version":"Falcon 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":474,"score":19.4,"version":"DeepSeek Coder V2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":475,"score":19.05,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":476,"score":18.9,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":477,"score":18.86,"version":"Gemma 3"},{"company":"cohere","family":"Command","model":"command-r-plus","place":478,"score":18.2,"version":"Command"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":479,"score":17.61,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":480,"score":16.4,"version":"Qwen3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":481,"score":16,"version":"KAT"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":482,"score":15.85,"version":"Llama 3.1"},{"company":"mistral","family":"Mistral Small","model":"mistral-small-1.0","place":483,"score":15.8,"version":"Mistral Small"},{"company":"aquif","family":"Aquif","model":"aquif-3-3.2b","place":484,"score":15.4,"version":"Aquif 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":485,"score":15.2,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":486,"score":15,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":487,"score":14.9,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":488,"score":14.41,"version":"LFM2"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":489,"score":14.3,"version":"Moonlight"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":490,"score":12.72,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":491,"score":12.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":492,"score":12.52,"version":"Qwen2.5"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b","place":493,"score":12.5,"version":"EXAONE 3.5"},{"company":"tencent","family":"HY","model":"hy-0.5b","place":494,"score":12.11,"version":"HY"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":495,"score":11.9,"version":"Moonlight"},{"company":"servicenow","family":"Apriel","model":"apriel-5b-instruct","place":496,"score":11.6,"version":"Apriel"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":497,"score":11.47,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":498,"score":11.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":499,"score":11.18,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":500,"score":10.8,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":501,"score":10.5,"version":"Qwen2.5"},{"company":"microsoft","family":"Phi","model":"phi-4-mini-3.8b","place":502,"score":10.4,"version":"Phi 4 mini"},{"company":"ai21","family":"Jamba","model":"jamba-1.6-mini","place":503,"score":9.9,"version":"Jamba 1.6"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":504,"score":9.78,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":505,"score":7.05,"version":"Falcon H1"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-360m","place":506,"score":4.8,"version":"MobileLLM R1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":507,"score":4.4,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":508,"score":4.27,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":509,"score":4.1,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":510,"score":4.08,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":511,"score":3.4,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":512,"score":3.13,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":513,"score":2.94,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":514,"score":2.74,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":515,"score":2.35,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":516,"score":2,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":517,"score":1.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":518,"score":1.2,"version":"Gemma 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-140m","place":519,"score":1,"version":"MobileLLM R1"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":520,"score":0.78,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-instruct","place":521,"score":0.7,"version":"SmolLM2"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":92,"topic":"Code generation","total_models":1613,"url":"https://livecodebench.github.io/"},{"bench_models":4,"description":"Korean translation of LiveCodeBench.","github":"","has_recent":0,"id":"livecodebench-ko","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LiveCodeBench-Ko","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":1,"score":66.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":2,"score":61.5,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":3,"score":51.2,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":4,"score":9.4,"version":"Kanana 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":66.3,"topic":"Code generation (Korean)","total_models":1613,"url":""},{"bench_models":13,"description":"LiveCodeBench Pro evaluates competitive programming performance across Codeforces, ICPC, and IOI contests. Elo rating, higher is better.","github":"","has_recent":0,"id":"livecodebench-pro","leaderboard":"https://livecodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"elo","name":"LiveCodeBench Pro","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":2887,"version":"Gemini 3.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":2439,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":3,"score":2393,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":2393,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":5,"score":2316,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":6,"score":2243,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":2114,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":8,"score":1930,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":1775,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":1765,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":11,"score":1418,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":12,"score":1418,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":1364,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":14,"score":1143,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":2887,"topic":"Competitive coding problems from Codeforces, ICPC, and IOI","total_models":1613,"url":"https://livecodebench.github.io/"},{"bench_models":13,"description":"LiveCodeBench Pro 2025 Q2 easy subset.","github":"","has_recent":0,"id":"livecodebench-pro-25q2-easy","leaderboard":"https://livecodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"LCB Pro 25Q2 (Easy)","paper":"","placements":[{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":1,"score":81.4,"version":"Nanbeige4.1"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":2,"score":68.9,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":3,"score":65.7,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":4,"score":64.8,"version":"Nemotron Cascade"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":5,"score":63.9,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":6,"score":60.8,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":7,"score":59.3,"version":"Nemotron Nano v2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":8,"score":53.6,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":9,"score":47.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":10,"score":46.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":11,"score":42.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":12,"score":41.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":13,"score":40.2,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":14,"score":40.2,"version":"Nanbeige4"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":15,"score":33,"version":"Qwen3"}],"stars":"","top_company":"boss","top_model":"Nanbeige4.1-3B","top_model_family":"Nanbeige","top_model_id":"nanbeige4.1-3b","top_model_version":"Nanbeige4.1","top_open_source":1,"top_percent":81.4,"topic":"Code generation","total_models":1613,"url":"https://livecodebench.github.io/"},{"bench_models":18,"description":"LiveCodeBench Pro 2025 Q2 medium subset.","github":"","has_recent":0,"id":"livecodebench-pro-25q2-med","leaderboard":"https://livecodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"LCB Pro 25Q2 (Med)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":1,"score":35.4,"version":"GPT-OSS"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":2,"score":28.1,"version":"Nanbeige4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":3,"score":27.9,"version":"DeepSeek V3.2"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":4,"score":25.9,"version":"K-EXAONE"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":5,"score":16,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":6,"score":10.5,"version":"Nemotron Cascade"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":7,"score":7,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":8,"score":6.4,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":9,"score":6.1,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":10,"score":5.3,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":11,"score":5.3,"version":"Nanbeige4"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":12,"score":4.8,"version":"Nemotron Nano v2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":13,"score":4.8,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":14,"score":3.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":15,"score":3.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":16,"score":3.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":17,"score":2.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":18,"score":2.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":19,"score":1.8,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":20,"score":1.8,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-OSS 120B (High)","top_model_family":"GPT","top_model_id":"gpt-oss-120b-high","top_model_version":"GPT-OSS","top_open_source":1,"top_percent":35.4,"topic":"Code generation","total_models":1613,"url":"https://livecodebench.github.io/"},{"bench_models":10,"description":"LiveCodeBench v3 snapshot measuring pass rates on streaming coding tasks.","github":"","has_recent":0,"id":"livecodebench-v3","leaderboard":"https://livecodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"LiveCodeBench v3","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":1,"score":90.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":2,"score":84.8,"version":"Qwen3 VL"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":3,"score":83.5,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":4,"score":81.9,"version":"OLMo 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":5,"score":79.5,"version":"DeepSeek R1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":6,"score":75.8,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":7,"score":49.9,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":8,"score":39,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":9,"score":28.7,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":10,"score":10.6,"version":"OLMo 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3 32B","top_model_family":"Qwen","top_model_id":"qwen-3-32b","top_model_version":"Qwen3","top_open_source":1,"top_percent":90.2,"topic":"Code generation","total_models":1613,"url":"https://livecodebench.github.io/"},{"bench_models":77,"description":"LiveCodeBench v5 snapshot covering Oct 2024-Feb 2025.","github":"","has_recent":0,"id":"livecodebench-v5","leaderboard":"https://livecodebench.github.io/leaderboard.html","lower_is_better":false,"metric":"percent","name":"LiveCodeBench v5 (2024.10-2025.02)","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-thinking","place":1,"score":86.2,"version":"IQuest Coder V1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":2,"score":83.8,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":3,"score":80.2,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":4,"score":77.5,"version":"Nemotron Cascade"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":5,"score":74.8,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":6,"score":74.5,"version":"Nemotron Cascade"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":7,"score":74.3,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":8,"score":70.7,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-3-beta-think","place":9,"score":70.6,"version":"Grok 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":70.4,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":11,"score":68.26,"version":"Nemotron Nano V2"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":12,"score":68.2,"version":"Nemotron Nano v2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":13,"score":67.66,"version":"Falcon H1R"},{"company":"openai","family":"o3","model":"o3-mini","place":14,"score":66.3,"version":"o3"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":15,"score":66.3,"version":"o3"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-ultra-253b-v1","place":16,"score":66.3,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0120","place":17,"score":65.9,"version":"0120"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":18,"score":65.87,"version":"Ministral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":19,"score":65.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":20,"score":65.2,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":21,"score":65,"version":"Motif 2"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":22,"score":64.9,"version":"Seed 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":23,"score":64.3,"version":"DeepSeek R1"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":24,"score":63.9,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":25,"score":63.4,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1","place":26,"score":62.3,"version":"MiniMax M1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":27,"score":61.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":28,"score":61.2,"version":"Qwen3"},{"company":"antgroup","family":"Ring","model":"ring-lite","place":29,"score":60.7,"version":"Ring Lite"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":30,"score":60.48,"version":"MiniCPM SALA"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":31,"score":60.1,"version":"Motif 2"},{"company":"magistral","family":"Magistral","model":"magistral-medium","place":32,"score":59.4,"version":"Magistral"},{"company":"mistral","family":"Magistral","model":"magistral-medium-2506","place":33,"score":59.4,"version":"Magistral 2506"},{"company":"xiaomi","family":"MiMo","model":"mimo-7b-rl","place":34,"score":57.8,"version":"MiMo 7B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-70b","place":35,"score":57.5,"version":"DeepSeek R1"},{"company":"nvidia","family":"AceReason","model":"acereason-nemotron-1.1","place":36,"score":57.2,"version":"1.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":37,"score":56.89,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":38,"score":56.89,"version":"MiniCPM 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":39,"score":56.6,"version":"Claude 4"},{"company":"weibo","family":"VibeThinker","model":"vibethinker-1.5b","place":40,"score":55.9,"version":"VibeThinker"},{"company":"mistral","family":"Magistral","model":"magistral-small-2506","place":41,"score":55.8,"version":"Magistral 2506"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":42,"score":55.7,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":43,"score":53.9,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-4-reasoning","place":44,"score":53.8,"version":"Phi 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":45,"score":52.1,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":46,"score":50.8,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-behemoth","place":47,"score":49.4,"version":"Llama 4"},{"company":"zai","family":"GLM","model":"glm-z1","place":48,"score":49.1,"version":"GLM Z1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":49,"score":48.6,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":50,"score":48.6,"version":"IQuest Coder V1"},{"company":"KunLun","family":"Skywork OR1","model":"skywork-or1-7b","place":51,"score":47.6,"version":"Skywork OR1"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":52,"score":47.2,"version":"EXAONE Deep"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":53,"score":46.1,"version":"Kimi Dev"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-v1","place":54,"score":45.5,"version":"Llama Nemotron v1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":55,"score":44.6,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":56,"score":43.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":57,"score":38.64,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":58,"score":36,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":59,"score":33.2,"version":"Qwen3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":60,"score":32.9,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":61,"score":30.5,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":62,"score":29.1,"version":"Mistral 3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":63,"score":27.6,"version":"SmolLM3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":64,"score":27.27,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":65,"score":26.48,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":66,"score":22.8,"version":"Qwen2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":67,"score":21.36,"version":"LFM2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":68,"score":19.2,"version":"SmollM3"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":69,"score":19.2,"version":"Seed Coder"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":70,"score":19.09,"version":"Gemma 3"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":71,"score":14.43,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":72,"score":14.4,"version":"Qwen2.5"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":73,"score":13.8,"version":"KAT"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":74,"score":13.2,"version":"DeepSeek Coder V2"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":75,"score":12.3,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":76,"score":11.48,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":77,"score":10.57,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":78,"score":4.43,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":79,"score":3.64,"version":"Llama 3.2"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Loop-Thinking","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-loop-thinking","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":86.2,"topic":"Code generation","total_models":1613,"url":"https://livecodebench.github.io/"},{"bench_models":18,"description":"A novel real-time evaluation framework and benchmark to stress‑test agents on complex, real‑world tasks.","github":"","has_recent":0,"id":"livemcp-101","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LiveMCP-101","paper":"https://arxiv.org/pdf/2508.15760","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":58.42,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":46.53,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":3,"score":43.56,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":4,"score":43.56,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":5,"score":41.58,"version":"Claude 4.1"},{"company":"openai","family":"o4","model":"o4-mini","place":6,"score":40.59,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":7,"score":35.64,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":8,"score":29.7,"version":"Claude 3.7"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":27.72,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":10,"score":22.77,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":11,"score":21.78,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":12,"score":18.81,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":13,"score":17.82,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":14,"score":10.89,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":15,"score":8.91,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":16,"score":3.96,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":17,"score":1.98,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":18,"score":0.99,"version":"Llama 3.1"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":58.42,"topic":"Agent real-time eval","total_models":1613,"url":""},{"bench_models":4,"description":"Live sports video understanding benchmark (3K).","github":"","has_recent":0,"id":"livesports-3k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LiveSports-3K","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":77.5,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":75.8,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":3,"score":74.9,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":74.5,"version":"Gemini 3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":77.5,"topic":"Sports video","total_models":1613,"url":""},{"bench_models":259,"description":"Chatbot Arena text evaluation (average win rate).","github":"","has_recent":0,"id":"lmarena-text","leaderboard":"https://lmarena.ai/leaderboard/text","lower_is_better":false,"metric":"elo","name":"LMArena Text","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":1455,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":1453,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":3,"score":1451,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805-thinking-16k","place":4,"score":1448,"version":"Claude 4.1"},{"company":"openai","family":"o3","model":"o3","place":5,"score":1444,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":1442,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":7,"score":1441,"version":"GPT-4o"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":8,"score":1440,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-4.5-preview-2025-02-27","place":9,"score":1440,"version":"GPT-4.5"},{"company":"openai","family":"GPT","model":"chatgpt-4o-latest-20250326","place":10,"score":1440,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":11,"score":1439,"version":"GPT-4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":12,"score":1439,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":13,"score":1439,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-max-preview","place":14,"score":1433,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":15,"score":1428,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-chat","place":16,"score":1426,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":17,"score":1425,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4","place":18,"score":1422,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":19,"score":1421,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":20,"score":1420,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":21,"score":1420,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514-thinking-16k","place":22,"score":1420,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-max-2025-09-23","place":23,"score":1418,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4-0709","place":24,"score":1418,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":25,"score":1418,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":26,"score":1417,"version":"DeepSeek R1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":27,"score":1416,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":28,"score":1415,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":29,"score":1415,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":30,"score":1415,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0711-preview","place":31,"score":1415,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus-thinking","place":32,"score":1414,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":33,"score":1411,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-09-2025","place":34,"score":1410,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-3-preview-02-24","place":35,"score":1410,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":36,"score":1409,"version":"GPT-4.1"},{"company":"mistral","family":"Mistral","model":"mistral-medium-2508","place":37,"score":1408,"version":"Mistral"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":38,"score":1407,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":39,"score":1407,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":40,"score":1402,"version":"Qwen3 Next"},{"company":"meituan","family":"LongCat","model":"longcat-flash-chat","place":41,"score":1399,"version":"LongCat-Flash"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":42,"score":1399,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-32k","place":43,"score":1398,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-no-thinking","place":44,"score":1398,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":45,"score":1397,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":46,"score":1394,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":47,"score":1393,"version":"GPT-5"},{"company":"mistral","family":"MAI","model":"mai-1-preview","place":48,"score":1392,"version":"MAI-1 Preview"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":49,"score":1391,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":50,"score":1391,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":51,"score":1388,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":52,"score":1387,"version":"Qwen3 VL"},{"company":"openai","family":"o1","model":"o1-preview","place":53,"score":1386,"version":"o1"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-t1-20250711","place":54,"score":1385,"version":"Hunyuan T1"},{"company":"anthropic","family":"Claude","model":"claude-3-7-sonnet-20250219-thinking-32k","place":55,"score":1385,"version":"Claude 3.7"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite-preview-09-2025","place":56,"score":1383,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":57,"score":1383,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":58,"score":1381,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-2505","place":59,"score":1380,"version":"Mistral"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-turbos-20250416","place":60,"score":1379,"version":"Hunyuan Turbos"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini-2025-04-14","place":61,"score":1377,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":62,"score":1372,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-max","place":63,"score":1371,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":64,"score":1370,"version":"Qwen3 Next"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":65,"score":1369,"version":"GLM 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1","place":66,"score":1369,"version":"MiniMax M1"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20241022","place":67,"score":1369,"version":"Claude 3.5"},{"company":"xai","family":"Grok","model":"grok-3-mini-high","place":68,"score":1362,"version":"Grok 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":69,"score":1362,"version":"Gemma 3"},{"company":"openai","family":"o3","model":"o3-mini-high","place":70,"score":1361,"version":"o3"},{"company":"xai","family":"Grok","model":"grok-3-mini-beta","place":71,"score":1357,"version":"Grok 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":72,"score":1356,"version":"DeepSeek V3"},{"company":"mistral","family":"Mistral","model":"mistral-small-2506","place":73,"score":1351,"version":"Mistral"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-lite-preview-02-05","place":74,"score":1351,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":75,"score":1350,"version":"Gemini 1.5"},{"company":"openai","family":"o3","model":"o3-mini","place":76,"score":1347,"version":"o3"},{"company":"cohere","family":"Command","model":"command-a-03-2025","place":77,"score":1347,"version":"Command"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":78,"score":1346,"version":"GLM 4.5V"},{"company":"stepfun","family":"Step","model":"step-3","place":79,"score":1346,"version":"Step 3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":80,"score":1346,"version":"GPT-OSS"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-turbos-20250226","place":81,"score":1344,"version":"Hunyuan Turbos"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-ultra-253b-v1","place":82,"score":1344,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":83,"score":1343,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-plus-0125","place":84,"score":1343,"version":"Qwen"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-05-13","place":85,"score":1343,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4-plus-0111","place":86,"score":1342,"version":"GLM 4"},{"company":"openai","family":"GPT","model":"gpt-5-nano-high","place":87,"score":1341,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20240620","place":88,"score":1340,"version":"Claude 3.5"},{"company":"google","family":"Gemma","model":"gemma-3-12b-it","place":89,"score":1339,"version":"Gemma 3"},{"company":"nvidia","family":"Llama","model":"nvidia-llama-3.3-nemotron-super-49b-v1.5","place":90,"score":1339,"version":"Llama 3.3"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-turbo-0110","place":91,"score":1337,"version":"Hunyuan Turbo"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct-bf16","place":92,"score":1334,"version":"Llama 3.1"},{"company":"openai","family":"o1","model":"o1-mini","place":93,"score":1333,"version":"o1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":94,"score":1333,"version":"QwQ"},{"company":"google","family":"Gemini","model":"gemini-advanced-0514","place":95,"score":1332,"version":"Gemini"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":96,"score":1332,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct-fp8","place":97,"score":1332,"version":"Llama 3.1"},{"company":"xai","family":"Grok","model":"grok-2-2024-08-13","place":98,"score":1332,"version":"Grok 2"},{"company":"stepfun","family":"Step","model":"step-2-16k-exp-202412","place":99,"score":1331,"version":"Step 2 16K EXP 202412"},{"company":"01.ai","family":"Yi","model":"yi-lightning","place":100,"score":1327,"version":"Yi"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-instruct","place":101,"score":1325,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":102,"score":1324,"version":"Qwen3"},{"company":"nvidia","family":"Llama","model":"llama-3.3-nemotron-49b-super-v1","place":103,"score":1323,"version":"Llama 3.3"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-large-2025-02-10","place":104,"score":1323,"version":"Hunyuan"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":105,"score":1322,"version":"GPT-4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5-1210","place":106,"score":1320,"version":"DeepSeek Coder"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":107,"score":1320,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-instruct","place":108,"score":1320,"version":"Llama 4"},{"company":"anthropic","family":"Claude","model":"claude-3-opus-20240229","place":109,"score":1320,"version":"Claude 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-001","place":110,"score":1320,"version":"Gemini 1.5"},{"company":"stepfun","family":"Step","model":"step-1o-turbo-202506","place":111,"score":1319,"version":"Step 1O Turbo 202506"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano-2025-04-14","place":112,"score":1318,"version":"GPT-4.1"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-experimental-chat-05-14","place":113,"score":1318,"version":"Amazon Nova"},{"company":"google","family":"Gemma","model":"gemma-3n-e4b-it","place":114,"score":1318,"version":"Gemma 3n"},{"company":"anthropic","family":"Claude","model":"claude-3-5-haiku-20241022","place":115,"score":1318,"version":"Claude 3.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":116,"score":1317,"version":"Llama 3.3"},{"company":"zai","family":"GLM","model":"glm-4-plus","place":117,"score":1316,"version":"GLM 4"},{"company":"qwen","family":"Qwen","model":"qwen-max-0919","place":118,"score":1315,"version":"Qwen"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":119,"score":1314,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4-1106-preview","place":120,"score":1313,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-plus-1127","place":121,"score":1312,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4-0125-preview","place":122,"score":1312,"version":"GPT-4"},{"company":"nvidia","family":"Athene","model":"athene-v2-chat","place":123,"score":1311,"version":"Athene V2"},{"company":"mistral","family":"Mistral","model":"mistral-large-2407","place":124,"score":1311,"version":"Mistral"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":125,"score":1310,"version":"Gemini 1.5"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-standard-2025-02-10","place":126,"score":1308,"version":"Hunyuan"},{"company":"xai","family":"Grok","model":"grok-2-mini-2024-08-13","place":127,"score":1305,"version":"Grok 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":128,"score":1304,"version":"DeepSeek V2.5"},{"company":"nexusflow","family":"Athene","model":"athene-70b-0725","place":129,"score":1303,"version":"Athene"},{"company":"mistral","family":"Mistral","model":"mistral-large-2411","place":130,"score":1303,"version":"Mistral Large"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":131,"score":1302,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":132,"score":1300,"version":"Qwen2.5"},{"company":"mistral","family":"Magistral","model":"magistral-medium-2506","place":133,"score":1297,"version":"Magistral 2506"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct-2503","place":134,"score":1297,"version":"Mistral 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-nemotron-70b-instruct","place":135,"score":1296,"version":"Llama 3.1"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-large-vision","place":136,"score":1292,"version":"Hunyuan Vision"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":137,"score":1292,"version":"Llama 3.1"},{"company":"ai21","family":"Jamba","model":"jamba-1.5-large","place":138,"score":1286,"version":"Jamba 1.5"},{"company":"reka","family":"Reka","model":"reka-core-20240904","place":139,"score":1286,"version":"Reka Core 20240904"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-pro-v1.0","place":140,"score":1286,"version":"Amazon Nova"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":141,"score":1285,"version":"Tulu 3"},{"company":"openai","family":"GPT","model":"gpt-4-0314","place":142,"score":1285,"version":"GPT-4"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":143,"score":1285,"version":"Gemma 2"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-51b-instruct","place":144,"score":1284,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-001","place":145,"score":1282,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet-20240229","place":146,"score":1280,"version":"Claude 3"},{"company":"princeton","family":"Gemma","model":"gemma-2-9b-it-simpo","place":147,"score":1277,"version":"Gemma 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-4-340b-instruct","place":148,"score":1277,"version":"Nemotron-4"},{"company":"cohere","family":"Command","model":"command-r-plus-08-2024","place":149,"score":1276,"version":"Command"},{"company":"meta","family":"Llama","model":"llama-3-70b-instruct","place":150,"score":1274,"version":"Llama 3"},{"company":"openai","family":"GPT","model":"gpt-4-0613","place":151,"score":1273,"version":"GPT-4"},{"company":"zai","family":"GLM","model":"glm-4-0520","place":152,"score":1272,"version":"GLM 4"},{"company":"reka","family":"Reka","model":"reka-flash-20240904","place":153,"score":1271,"version":"Reka Flash 20240904"},{"company":"mistral","family":"Mistral","model":"mistral-small-24b-instruct-2501","place":154,"score":1271,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":155,"score":1267,"version":"Qwen2.5"},{"company":"cohere","family":"Aya","model":"c4ai-aya-expanse-32b","place":156,"score":1265,"version":"Aya Expanse"},{"company":"cohere","family":"Command","model":"command-r-plus","place":157,"score":1262,"version":"Command"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":158,"score":1262,"version":"Gemma 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2","place":159,"score":1261,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen2-72b-instruct","place":160,"score":1261,"version":"Qwen2"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku-20240307","place":161,"score":1259,"version":"Claude 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-001","place":162,"score":1258,"version":"Gemini 1.5"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-lite-v1.0","place":163,"score":1258,"version":"Amazon Nova"},{"company":"microsoft","family":"Phi","model":"phi-4","place":164,"score":1253,"version":"Phi 3.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":165,"score":1252,"version":"OLMo 2"},{"company":"cohere","family":"Command","model":"command-r-08-2024","place":166,"score":1250,"version":"Command"},{"company":"mistral","family":"Mistral","model":"mistral-large-2402","place":167,"score":1241,"version":"Mistral"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-micro-v1.0","place":168,"score":1240,"version":"Amazon Nova"},{"company":"ai21","family":"Jamba","model":"jamba-1.5-mini","place":169,"score":1237,"version":"Jamba 1.5"},{"company":"mistral","family":"Ministral","model":"ministral-8b-2410","place":170,"score":1235,"version":"Ministral"},{"company":"qwen","family":"Qwen","model":"qwen1.5-110b-chat","place":171,"score":1233,"version":"Qwen1.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-72b-chat","place":172,"score":1232,"version":"Qwen1.5"},{"company":"reka","family":"Reka","model":"reka-flash-21b-20240226-online","place":173,"score":1232,"version":"Reka Flash 21B 20240226 Online"},{"company":"google","family":"Gemini","model":"gemini-pro-dev-api","place":174,"score":1232,"version":"Gemini"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-standard-256k","place":175,"score":1231,"version":"Hunyuan"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b-instruct-v0.1","place":176,"score":1229,"version":"Mixtral v0.1"},{"company":"cohere","family":"Command","model":"command-r","place":177,"score":1227,"version":"Command"},{"company":"reka","family":"Reka","model":"reka-flash-21b-20240226","place":178,"score":1225,"version":"Reka Flash 21B 20240226"},{"company":"mistral","family":"Mistral","model":"mistral-medium","place":179,"score":1223,"version":"Mistral"},{"company":"cohere","family":"Aya","model":"c4ai-aya-expanse-8b","place":180,"score":1222,"version":"Aya Expanse"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0125","place":181,"score":1222,"version":"GPT-3.5"},{"company":"meta","family":"Llama","model":"llama-3-8b-instruct","place":182,"score":1222,"version":"Llama 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":183,"score":1221,"version":"Tulu 3"},{"company":"google","family":"Gemini","model":"gemini-pro","place":184,"score":1219,"version":"Gemini"},{"company":"01ai","family":"Yi 1.5","model":"yi-1.5-34b-chat","place":185,"score":1213,"version":"Yi 1.5"},{"company":"huggingface","family":"Zephyr","model":"zephyr-orpo-141b-a35b-v0.1","place":186,"score":1212,"version":"Zephyr ORPO 141B A35B v0.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":187,"score":1210,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":188,"score":1209,"version":"Granite 3.1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":189,"score":1204,"version":"Qwen1.5"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-1106","place":190,"score":1199,"version":"GPT-3.5"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":191,"score":1197,"version":"Phi 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b-instruct-v0.1","place":192,"score":1197,"version":"Mixtral v0.1"},{"company":"google","family":"Gemma","model":"gemma-2-2b-it","place":193,"score":1196,"version":"Gemma 2"},{"company":"databricks","family":"DBRX","model":"dbrx-instruct-preview","place":194,"score":1194,"version":"DBRX"},{"company":"internlm","family":"Intern","model":"internlm2_5-20b-chat","place":195,"score":1192,"version":"InternLM2.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b-chat","place":196,"score":1192,"version":"Qwen1.5"},{"company":"wizardlm","family":"WizardLM","model":"wizardlm-70b","place":197,"score":1184,"version":"WizardLM-70B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-llm-67b-chat","place":198,"score":1183,"version":"DeepSeek"},{"company":"01ai","family":"Yi","model":"yi-34b-chat","place":199,"score":1183,"version":"Yi 34B Chat"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5-0106","place":200,"score":1181,"version":"OpenChat 3.5 0106"},{"company":"IBM","family":"Granite","model":"granite-3.0-8b-instruct","place":201,"score":1181,"version":"Granite 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":202,"score":1180,"version":"Granite 3.1"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5","place":203,"score":1180,"version":"OpenChat 3.5"},{"company":"snowflake","family":"Arctic","model":"snowflake-arctic-instruct","place":204,"score":1179,"version":"Arctic Instruct"},{"company":"ai2","family":"Tulu","model":"tulu-2-dpo-70b","place":205,"score":1178,"version":"Tulu 2 DPO 70B"},{"company":"google","family":"Gemma","model":"gemma-1.1-7b-it","place":206,"score":1177,"version":"gemma-1.1"},{"company":"openhermes","family":"OpenHermes","model":"openhermes-2.5-mistral-7b","place":207,"score":1175,"version":"OpenHermes 2.5"},{"company":"lmsys","family":"Vicuna","model":"vicuna-33b","place":208,"score":1172,"version":"Vicuna-33B"},{"company":"berkeley","family":"Starling","model":"starling-lm-7b-beta","place":209,"score":1172,"version":"Starling LM"},{"company":"microsoft","family":"Phi","model":"phi-3-small-8k-instruct","place":210,"score":1171,"version":"Phi-3"},{"company":"meta","family":"Llama","model":"llama-2-70b-chat","place":211,"score":1171,"version":"Llama 2"},{"company":"berkeley","family":"Starling","model":"starling-lm-7b-alpha","place":212,"score":1167,"version":"Starling LM"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":213,"score":1166,"version":"Llama 3.2"},{"company":"nousresearch","family":"Nous Hermes","model":"nous-hermes-2-mixtral-8x7b-dpo","place":214,"score":1165,"version":"Nous Hermes 2"},{"company":"qwen","family":"QwQ","model":"qwq-32b-preview","place":215,"score":1161,"version":"QwQ"},{"company":"nvidia","family":"Llama","model":"llama2-70b-steerlm-chat","place":216,"score":1157,"version":"Llama 2"},{"company":"IBM","family":"Granite","model":"granite-3.0-2b-instruct","place":217,"score":1155,"version":"Granite 3.0"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":218,"score":1153,"version":"Solar Mini"},{"company":"cognitivecomputations","family":"Dolphin","model":"dolphin-2.2.1-mistral-7b","place":219,"score":1151,"version":"Dolphin"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":220,"score":1150,"version":"Mistral v0.2"},{"company":"mosaicml","family":"MPT","model":"mpt-30b-chat","place":221,"score":1149,"version":"MPT"},{"company":"wizardlm","family":"WizardLM","model":"wizardlm-13b","place":222,"score":1149,"version":"WizardLM-13B"},{"company":"tii","family":"Falcon","model":"falcon-180b-chat","place":223,"score":1145,"version":"Falcon"},{"company":"qwen","family":"Qwen","model":"qwen1.5-7b-chat","place":224,"score":1143,"version":"Qwen1.5"},{"company":"microsoft","family":"Phi","model":"phi-3-mini-4k-instruct-june-2024","place":225,"score":1142,"version":"Phi-3"},{"company":"meta","family":"Llama","model":"llama-2-13b-chat","place":226,"score":1142,"version":"Llama 2"},{"company":"lmsys","family":"Vicuna-13B","model":"vicuna-13b","place":227,"score":1140,"version":"Vicuna-13B"},{"company":"qwen","family":"Qwen","model":"qwen-14b-chat","place":228,"score":1137,"version":"Qwen"},{"company":"meta","family":"CodeLlama","model":"codellama-34b-instruct","place":229,"score":1136,"version":"CodeLlama"},{"company":"google","family":"PaLM","model":"palm-2","place":230,"score":1134,"version":"PaLM 2"},{"company":"google","family":"Gemma","model":"gemma-7b-it","place":231,"score":1132,"version":"Gemma"},{"company":"huggingface","family":"Zephyr","model":"zephyr-7b-beta","place":232,"score":1131,"version":"Zephyr 7B Beta"},{"company":"microsoft","family":"Phi","model":"phi-3-mini-128k-instruct","place":233,"score":1130,"version":"Phi 3.5"},{"company":"huggingface","family":"Zephyr","model":"zephyr-7b-alpha","place":234,"score":1128,"version":"Zephyr 7B Alpha"},{"company":"microsoft","family":"Phi-3-mini-4k-instruct","model":"phi-3-mini-4k-instruct","place":235,"score":1128,"version":"Phi-3-mini-4k-instruct"},{"company":"timdettmers","family":"Guanaco","model":"guanaco-33b","place":236,"score":1127,"version":"Guanaco-33B"},{"company":"togethercomputer","family":"StripedHyena","model":"stripedhyena-nous-7b","place":237,"score":1119,"version":"StripedHyena Nous 7B"},{"company":"codellama","family":"CodeLlama","model":"codellama-70b-instruct","place":238,"score":1118,"version":"CodeLlama"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":239,"score":1117,"version":"SmolLM2"},{"company":"lmsys","family":"Vicuna-7B","model":"vicuna-7b","place":240,"score":1114,"version":"Vicuna-7B"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":241,"score":1112,"version":"Llama 3.2"},{"company":"google","family":"gemma-1.1-2b-it","model":"gemma-1.1-2b-it","place":242,"score":1111,"version":"gemma-1.1"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct","place":243,"score":1109,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-2-7b-chat","place":244,"score":1108,"version":"Llama 2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-4b-chat","place":245,"score":1089,"version":"Qwen1.5"},{"company":"google","family":"Gemma","model":"gemma-2b-it","place":246,"score":1088,"version":"Gemma"},{"company":"allenai","family":"OLMo","model":"olmo-7b-instruct","place":247,"score":1075,"version":"OLMo"},{"company":"ucberkeley","family":"Koala","model":"koala-13b","place":248,"score":1069,"version":"Koala"},{"company":"stanford","family":"Alpaca","model":"alpaca-13b","place":249,"score":1063,"version":"Alpaca"},{"company":"nomic","family":"GPT4All","model":"gpt4all-13b-snoozy","place":250,"score":1062,"version":"GPT4All"},{"company":"mosaicml","family":"MPT","model":"mpt-7b-chat","place":251,"score":1059,"version":"MPT"},{"company":"zai","family":"ChatGLM","model":"chatglm3-6b","place":252,"score":1056,"version":"ChatGLM3"},{"company":"rwkv","family":"RWKV","model":"rwkv-4-raven-14b","place":253,"score":1040,"version":"RWKV-4 Raven"},{"company":"zai","family":"ChatGLM","model":"chatglm2-6b","place":254,"score":1024,"version":"ChatGLM2"},{"company":"oasst","family":"OASST","model":"oasst-pythia-12b","place":255,"score":1019,"version":"OASST-Pythia"},{"company":"zai","family":"ChatGLM","model":"chatglm-6b","place":256,"score":994,"version":"ChatGLM"},{"company":"lmsys","family":"FastChat","model":"fastchat-t5-3b","place":257,"score":989,"version":"FastChat"},{"company":"databricks","family":"Dolly","model":"dolly-v2-12b","place":258,"score":975,"version":"Dolly v2"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":259,"score":966,"version":"Llama 1"},{"company":"stabilityai","family":"StableLM","model":"stablelm-tuned-alpha-7b","place":260,"score":950,"version":"Tuned Alpha 7B"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":1455,"topic":"Crowd eval (text)","total_models":1613,"url":""},{"bench_models":79,"description":"Chatbot Arena vision evaluation leaderboard (ELO ratings).","github":"","has_recent":0,"id":"lmarena-vision","leaderboard":"https://lmarena.ai/leaderboard/vision","lower_is_better":false,"metric":"elo","name":"LMArena Vision","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":1242,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"chatgpt-4o-latest-20250326","place":2,"score":1235,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4.5-preview-2025-02-27","place":3,"score":1222,"version":"GPT-4.5"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":4,"score":1217,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-chat","place":5,"score":1213,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":6,"score":1210,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":7,"score":1208,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-09-2025","place":8,"score":1207,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":9,"score":1205,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514-thinking-16k","place":10,"score":1204,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514-thinking-32k","place":11,"score":1200,"version":"Claude 4"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":12,"score":1196,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":13,"score":1195,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini-2025-04-14","place":14,"score":1193,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-3-7-sonnet-20250219-thinking-32k","place":15,"score":1191,"version":"Claude 3.7"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":16,"score":1189,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":17,"score":1185,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":18,"score":1184,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":19,"score":1182,"version":"Claude 4"},{"company":"xai","family":"Grok","model":"grok-4-0709","place":20,"score":1179,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite-preview-09-2025","place":21,"score":1175,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":22,"score":1174,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":23,"score":1174,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen-vl-max-2025-08-13","place":24,"score":1173,"version":"Qwen VL Max"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-05-13","place":25,"score":1158,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":26,"score":1157,"version":"Claude 3.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":27,"score":1156,"version":"GLM 4.5V"},{"company":"stepfun","family":"Step","model":"step-1o-turbo-202506","place":28,"score":1149,"version":"Step 1O Turbo 202506"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":29,"score":1149,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-2505","place":30,"score":1148,"version":"Mistral"},{"company":"openai","family":"GPT","model":"gpt-5-nano-high","place":31,"score":1144,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20240620","place":32,"score":1143,"version":"Claude 3.5"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-large-vision","place":33,"score":1141,"version":"Hunyuan Vision"},{"company":"stepfun","family":"Step","model":"step-3","place":34,"score":1139,"version":"Step 3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-2508","place":35,"score":1138,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-instruct","place":36,"score":1138,"version":"Llama 4"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":37,"score":1134,"version":"Gemini 1.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-2506","place":38,"score":1132,"version":"Mistral"},{"company":"anthropic","family":"Claude","model":"claude-3-5-haiku-20241022","place":39,"score":1129,"version":"Claude 3.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-lite-preview-02-05","place":40,"score":1129,"version":"Gemini 2.0"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-instruct","place":41,"score":1122,"version":"Llama 4"},{"company":"stepfun","family":"Step","model":"step-1o-vision-32k-highres","place":42,"score":1119,"version":"Step 1O Vision"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct-2503","place":43,"score":1118,"version":"Mistral 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":44,"score":1117,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":45,"score":1115,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-001","place":46,"score":1113,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-32b-instruct","place":47,"score":1111,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":48,"score":1109,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":49,"score":1093,"version":"GPT-4o"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":50,"score":1089,"version":"Pixtral"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano-2025-04-14","place":51,"score":1085,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen-vl-max-1119","place":52,"score":1081,"version":"Qwen VL Max"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":53,"score":1081,"version":"Qwen2 VL"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-001","place":54,"score":1066,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus-20240229","place":55,"score":1060,"version":"Claude 3"},{"company":"stepfun","family":"Step","model":"step-1v-32k","place":56,"score":1057,"version":"Step 1V 32K"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-001","place":57,"score":1056,"version":"Gemini 1.5"},{"company":"allenai","family":"Molmo","model":"molmo-72b-0924","place":58,"score":1044,"version":"Molmo"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-standard-vision-2024-12-31","place":59,"score":1040,"version":"Hunyuan Vision"},{"company":"meta","family":"Llama","model":"llama-3.2-vision-90b-instruct","place":60,"score":1028,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b-instruct","place":61,"score":1028,"version":"Qwen2 VL"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b-2409","place":62,"score":1020,"version":"Pixtral"},{"company":"opengvlab","family":"Intern","model":"internvl2-26b","place":63,"score":1020,"version":"InternVL 2"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-lite-v1.0","place":64,"score":1017,"version":"Amazon Nova"},{"company":"amazon","family":"Amazon Nova","model":"amazon-nova-pro-v1.0","place":65,"score":1016,"version":"Amazon Nova"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet-20240229","place":66,"score":1016,"version":"Claude 3"},{"company":"01ai","family":"Yi","model":"yi-vision","place":67,"score":1000,"version":"Yi Vision"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku-20240307","place":68,"score":999,"version":"Claude 3"},{"company":"cohere","family":"Aya","model":"c4ai-aya-vision-32b","place":69,"score":996,"version":"Aya Vision"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d-0924","place":70,"score":992,"version":"Molmo"},{"company":"meta","family":"Llama","model":"llama-3.2-vision-11b-instruct","place":71,"score":987,"version":"Llama 3.2"},{"company":"nvidia","family":"NVILA","model":"nvila-internal-15b-v1","place":72,"score":984,"version":"NVILA"},{"company":"llava","family":"LLaVA","model":"llava-onevision-qwen2-72b-ov","place":73,"score":977,"version":"LLaVA OneVision"},{"company":"microsoft","family":"LLaVA","model":"llava-1.6-34b","place":74,"score":963,"version":"LLaVA 1.6"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-2_6","place":75,"score":961,"version":"MiniCPM V 2.6"},{"company":"zai","family":"CogVLM2","model":"cogvlm2-llama3-chat-19b","place":76,"score":960,"version":"CogVLM2"},{"company":"opengvlab","family":"Intern","model":"internvl2-4b","place":77,"score":953,"version":"InternVL 2"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-instruct","place":78,"score":919,"version":"Phi 3.5"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-128k-instruct","place":79,"score":881,"version":"Phi 3"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":1242,"topic":"Crowd eval (vision)","total_models":1613,"url":""},{"bench_models":21,"description":"Tests whether small open-weight models can reliably decide when to call tools and when not to. Agent Score = (Action x 0.4) + (Restraint x 0.3) + (Wrong-Tool-Avoidance x 0.3).","github":"https://github.com/MikeVeerman/tool-calling-benchmark","has_recent":0,"id":"local-agent-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Local Agent Bench","paper":"","placements":[{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":1,"score":88,"version":"LFM2.5"},{"company":"microsoft","family":"Phi","model":"phi-4-mini-3.8b","place":2,"score":88,"version":"Phi 4 mini"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":3,"score":88,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":4,"score":88,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":5,"score":84,"version":"Qwen2.5"},{"company":"microsoft","family":"BitNet","model":"bitnet-2b-4t","place":6,"score":81,"version":"BitNet b1.58"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":7,"score":80,"version":"Ministral 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":8,"score":74,"version":"SmolLM2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":9,"score":72,"version":"DeepSeek R1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":10,"score":71,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":11,"score":67,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":12,"score":67,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":13,"score":67,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":14,"score":66,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":15,"score":64,"version":"Qwen2.5"},{"company":"meetkai","family":"Gemma","model":"functiongemma","place":16,"score":64,"version":"FunctionGemma"},{"company":"microsoft","family":"BitNet","model":"bitnet-3b","place":17,"score":60,"version":"BitNet b1.58"},{"company":"menlo","family":"Jan","model":"jan-v3","place":18,"score":56,"version":"Jan v3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":19,"score":55,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":20,"score":48,"version":"Granite 3.3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":21,"score":43,"version":"Llama 3.2"}],"stars":"","top_company":"liquidai","top_model":"LFM2.5 1.2B Instruct","top_model_family":"LFM","top_model_id":"lfm2.5-1.2b-instruct","top_model_version":"LFM2.5","top_open_source":1,"top_percent":88,"topic":"Tool calling judgment","total_models":1613,"url":""},{"bench_models":20,"description":"Visual logic and pattern reasoning tasks requiring compositional and spatial understanding.","github":"","has_recent":0,"id":"logicvista","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LogicVista","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":80.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":73.8,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":73.4,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":73.4,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":71.8,"version":"GPT-5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":6,"score":71.6,"version":"Ovis 2.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":70,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":8,"score":69.1,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":9,"score":65.3,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":10,"score":64,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":11,"score":62.4,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":12,"score":60.4,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":13,"score":60.2,"version":"Step 3"},{"company":"tencent","family":"R","model":"r-4b-base","place":14,"score":59.1,"version":"R"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":15,"score":56.4,"version":"InternVL 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":16,"score":51.4,"version":"Kimi VL"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":17,"score":50.6,"version":"Keye-VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":18,"score":47.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":19,"score":44.5,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3-8b","place":20,"score":43.6,"version":"InternVL 3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":80.8,"topic":"Visual logical reasoning","total_models":1613,"url":""},{"bench_models":8,"description":"Reading comprehension with logical reasoning.","github":"","has_recent":0,"id":"logiqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LogiQA","paper":"","placements":[{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":1,"score":23.5,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":2,"score":22.4,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":3,"score":22.1,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":4,"score":21.5,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":5,"score":21.4,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":6,"score":21.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":7,"score":21,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":8,"score":20.9,"version":"Pythia"}],"stars":"138","top_company":"eleutherai","top_model":"Pythia 70M","top_model_family":"Pythia","top_model_id":"pythia-70m","top_model_version":"Pythia","top_open_source":1,"top_percent":23.5,"topic":"Logical reasoning","total_models":1613,"url":"https://github.com/lgw863/LogiQA-dataset"},{"bench_models":4,"description":"Long-context understanding across tasks.","github":"","has_recent":0,"id":"longbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LongBench","paper":"","placements":[{"company":"ai21","family":"Jamba","model":"jamba-mini-1.6","place":1,"score":32,"version":"Jamba Mini"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":2,"score":17.7,"version":"Llama 3.1"},{"company":"mistral","family":"Ministral","model":"ministral-8b","place":3,"score":17.5,"version":"Ministral"},{"company":"cohere","family":"Command","model":"command-r7b","place":4,"score":2,"version":"Command"}],"stars":"957","top_company":"ai21","top_model":"Jamba Mini 1.6","top_model_family":"Jamba","top_model_id":"jamba-mini-1.6","top_model_version":"Jamba Mini","top_open_source":0,"top_percent":32,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/THUDM/LongBench"},{"bench_models":54,"description":"Next-generation LongBench v2 long-context evaluation benchmark.","github":"","has_recent":0,"id":"longbench-v2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"longbench-v2","paper":"https://arxiv.org/abs/2412.15204","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":68.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":67.4,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":65.6,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":65,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":65,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":6,"score":64.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":64.4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":64,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":9,"score":63.8,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":10,"score":63.2,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":11,"score":63.2,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":62,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":13,"score":61.8,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":14,"score":61.5,"version":"MiniMax M1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":15,"score":61,"version":"MiniMax M1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":16,"score":61,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":17,"score":61,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":18,"score":60.9,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":19,"score":60.6,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":20,"score":60.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":21,"score":60.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":22,"score":60.2,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":23,"score":59.8,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":24,"score":59.64,"version":"Kimi K2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":25,"score":59.6,"version":"Seed 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":26,"score":59,"version":"Qwen3.5"},{"company":"openai","family":"o3","model":"o3","place":27,"score":58.8,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":28,"score":58.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":29,"score":58.3,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":30,"score":56.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":31,"score":56.7,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":32,"score":55.8,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":33,"score":55.6,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":34,"score":55.5,"version":"Gemini 2.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":35,"score":55.3,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":36,"score":54.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":37,"score":54.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":38,"score":54.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":39,"score":54.3,"version":"GPT-4.1"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":40,"score":53.68,"version":"Ling 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":41,"score":52.88,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":42,"score":52.5,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":43,"score":52.5,"version":"Claude 4"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":44,"score":52.3,"version":"Seed 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":45,"score":52.1,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":46,"score":51.89,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":47,"score":51.1,"version":"DeepSeek V3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":48,"score":50.3,"version":"Ling 1T"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":49,"score":50.1,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":50,"score":49.4,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":51,"score":49.1,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":52,"score":48.2,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":53,"score":47.2,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":54,"score":47.1,"version":"OLMo 3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":55,"score":45.3,"version":"openPangu-R"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":56,"score":45.1,"version":"Kimi K2"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-4b-instruct","place":57,"score":44.1,"version":"Hunyuan"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":58,"score":42.8,"version":"OLMo 3"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":59,"score":42.6,"version":"K2 V2"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":60,"score":41.3,"version":"K2 V2"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":61,"score":40.7,"version":"K2 V2"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":68.2,"topic":"Long-context eval","total_models":1613,"url":""},{"bench_models":11,"description":"Long-context factuality eval focused on conceptual statements; lower is better.","github":"","has_recent":0,"id":"longfact-concepts","leaderboard":"","lower_is_better":true,"metric":"percent","name":"LongFact-Concepts","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":0.7,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":4.5,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":3,"score":91.4,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":4,"score":92.4,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":5,"score":92.8,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":98.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":98.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":98.6,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":98.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":10,"score":99.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":11,"score":99.7,"version":"GPT-5.2"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":0.7,"topic":"Hallucination rate on open-source prompts","total_models":1613,"url":""},{"bench_models":11,"description":"Long-context factuality eval focused on object/entity references; lower is better.","github":"","has_recent":0,"id":"longfact-objects","leaderboard":"","lower_is_better":true,"metric":"percent","name":"LongFact-Objects","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":0.8,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":5.1,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":3,"score":87,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":4,"score":92.2,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":5,"score":92.9,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":6,"score":97.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":98.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":98.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":99,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":10,"score":99.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":11,"score":99.2,"version":"GPT-5"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":0.8,"topic":"Hallucination rate on open-source prompts","total_models":1613,"url":""},{"bench_models":9,"description":"LongText-Bench English subset score for text rendering.","github":"","has_recent":0,"id":"longtext-bench-en","leaderboard":"","lower_is_better":false,"metric":"score","name":"LongText-Bench EN","paper":"","placements":[{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":1,"score":0.989,"version":"Seedream"},{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":2,"score":0.981,"version":"Nano Banana"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":3,"score":0.956,"version":"GPT Image 1"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":4,"score":0.956,"version":"Qwen-Image"},{"company":"zai","family":"GLM","model":"glm-image","place":5,"score":0.952,"version":"GLM-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":6,"score":0.943,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image","place":7,"score":0.935,"version":"Z-Image"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":8,"score":0.921,"version":"Seedream"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":9,"score":0.917,"version":"Z-Image"}],"stars":"","top_company":"bytedance","top_model":"Seedream 4.5","top_model_family":"Seedream","top_model_id":"seedream-4.5","top_model_version":"Seedream","top_open_source":0,"top_percent":0.989,"topic":"Text rendering","total_models":1613,"url":""},{"bench_models":9,"description":"LongText-Bench Chinese subset score for text rendering.","github":"","has_recent":0,"id":"longtext-bench-zh","leaderboard":"","lower_is_better":false,"metric":"score","name":"LongText-Bench ZH","paper":"","placements":[{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":1,"score":0.987,"version":"Seedream"},{"company":"zai","family":"GLM","model":"glm-image","place":2,"score":0.979,"version":"GLM-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":3,"score":0.965,"version":"Qwen-Image"},{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":4,"score":0.949,"version":"Nano Banana"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":5,"score":0.946,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image","place":6,"score":0.936,"version":"Z-Image"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":7,"score":0.926,"version":"Seedream"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":8,"score":0.926,"version":"Z-Image"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":9,"score":0.619,"version":"GPT Image 1"}],"stars":"","top_company":"bytedance","top_model":"Seedream 4.5","top_model_family":"Seedream","top_model_id":"seedream-4.5","top_model_version":"Seedream","top_open_source":0,"top_percent":0.987,"topic":"Text rendering","total_models":1613,"url":""},{"bench_models":8,"description":"Long video understanding and QA benchmark.","github":"","has_recent":0,"id":"longvideobench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LongVideoBench","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":1,"score":79.8,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":77.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":77.6,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":77.4,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":76.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":6,"score":76.5,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":7,"score":74,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":8,"score":67.2,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":9,"score":65.6,"version":"Qwen3 VL"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2.5 Thinking","top_model_family":"Kimi","top_model_id":"kimi-k2.5-thinking","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":79.8,"topic":"Long video QA","total_models":1613,"url":""},{"bench_models":12,"description":"Long-form financial question answering benchmark.","github":"","has_recent":0,"id":"lpfqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LPFQA","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":1,"score":54.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":54.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":3,"score":54.4,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":52.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":5,"score":52.6,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":6,"score":51.6,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":51.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":51.1,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":9,"score":50.9,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":10,"score":50.7,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":49.5,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":12,"score":49.1,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":47.7,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":14,"score":47.2,"version":"Seed 2.0"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.5","top_model_family":"Claude","top_model_id":"claude-sonnet-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":54.9,"topic":"Finance QA","total_models":1613,"url":""},{"bench_models":29,"description":"Long video understanding benchmark (LVBench).","github":"","has_recent":0,"id":"lvbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"LVBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":76.2,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":2,"score":75.9,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":75.9,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":75.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":74.4,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":73.7,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":7,"score":73.6,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":73.5,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":73.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":73,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":11,"score":73,"version":"Seed 1.8"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":12,"score":71.4,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":13,"score":68,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":14,"score":64.6,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":15,"score":64.6,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":16,"score":63.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":17,"score":63.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":18,"score":63.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":19,"score":62.5,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":20,"score":59.5,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":21,"score":58,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":22,"score":57.3,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":23,"score":56.2,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":24,"score":53.8,"version":"GLM 4.5V"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":25,"score":50.6,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":26,"score":49.5,"version":"GLM 4.6V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":27,"score":47.6,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":28,"score":47.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":29,"score":47.3,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":30,"score":45.9,"version":"Gemma 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":31,"score":44,"version":"GLM 4.1V"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":76.2,"topic":"Video understanding","total_models":1613,"url":""},{"bench_models":4,"description":"Chinese-language M3GIA benchmark covering grounded multimodal question answering.","github":"","has_recent":0,"id":"m3gia-cn","leaderboard":"","lower_is_better":false,"metric":"percent","name":"M3GIA (CN)","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":1,"score":91.2,"version":"Seed 1.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":2,"score":90.85,"version":"dots.vlm1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":89.54,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":4,"score":88.24,"version":"Qwen2.5 VL"}],"stars":"","top_company":"bytedance","top_model":"Seed1.5-VL-Thinking","top_model_family":"Seed","top_model_id":"seed1.5-vl-thinking","top_model_version":"Seed 1.5","top_open_source":0,"top_percent":91.2,"topic":"Chinese multimodal QA","total_models":1613,"url":""},{"bench_models":15,"description":"Benchmark for deceptive or manipulative behavior in social interactions.","github":"","has_recent":0,"id":"machiavelli","leaderboard":"","lower_is_better":true,"metric":"percent","name":"Machiavelli","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":1,"score":52.2,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":2,"score":52.6,"version":"Grok 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":53.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":53.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":5,"score":54.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":55.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":7,"score":56.1,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":8,"score":57.1,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":9,"score":57.7,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":10,"score":57.9,"version":"GPT-5.1"},{"company":"xai","family":"Grok","model":"grok-4","place":11,"score":58.6,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":12,"score":62.4,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":62.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":14,"score":62.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":15,"score":71.8,"version":"Gemini 2.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Haiku 4.5","top_model_family":"Claude","top_model_id":"claude-4.5-haiku","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":52.2,"topic":"Deception / safety","total_models":1613,"url":""},{"bench_models":2,"description":"Adversarial benchmark testing model robustness against manipulation attempts. Lower is better.","github":"","has_recent":0,"id":"makemesay","leaderboard":"","lower_is_better":true,"metric":"percent","name":"MakeMeSay","paper":"","placements":[{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":1,"score":0,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":2,"score":13,"version":"Grok 4"}],"stars":"","top_company":"xai","top_model":"Grok 4.1 (Thinking)","top_model_family":"Grok","top_model_id":"grok-4.1-thinking","top_model_version":"Grok 4.1","top_open_source":0,"top_percent":0,"topic":"Adversarial robustness","total_models":1613,"url":""},{"bench_models":11,"description":"Multimodal reasoning and instruction following benchmark (Mantis).","github":"","has_recent":0,"id":"mantis","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Mantis","paper":"","placements":[{"company":"rednote","family":"dots","model":"dots-vlm1","place":1,"score":86.18,"version":"dots.vlm1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":84.33,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":82.3,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":4,"score":79.26,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":5,"score":78.8,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":6,"score":78.3,"version":"MiMo VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":7,"score":75.6,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":8,"score":75.1,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":9,"score":74.7,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":10,"score":72.8,"version":"InternVL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":11,"score":70,"version":"Gemma 3"}],"stars":"","top_company":"rednote","top_model":"dots.vlm1","top_model_family":"dots","top_model_id":"dots-vlm1","top_model_version":"dots.vlm1","top_open_source":1,"top_percent":86.18,"topic":"Multimodal reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Multilingual variant of Arena-Hard evaluating chat quality across languages.","github":"","has_recent":0,"id":"marena-hard","leaderboard":"","lower_is_better":false,"metric":"percent","name":"mArenaHard","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":1,"score":70.1,"version":"Qwen3"},{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":2,"score":67.4,"version":"Aya"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":3,"score":65.8,"version":"Gemma 3"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":4,"score":63.7,"version":"Ministral 3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-4B","top_model_family":"Qwen","top_model_id":"qwen3-4b","top_model_version":"Qwen3","top_open_source":1,"top_percent":70.1,"topic":"Chat ability (multilingual)","total_models":1613,"url":""},{"bench_models":12,"description":"Instruction-following benchmark with complex tasks.","github":"","has_recent":0,"id":"mars-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MARS-Bench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":87.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":87.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":85.6,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":4,"score":85.6,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":5,"score":84.6,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":80.8,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":7,"score":80.5,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":8,"score":77.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":73.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":10,"score":72.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":72.5,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":12,"score":70.1,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":13,"score":66.1,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":14,"score":62.4,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":87.9,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":66,"description":"Model behavior safety assessment via red-teaming scenarios.","github":"https://github.com/centerforaisafety/mask","has_recent":0,"id":"mask","leaderboard":"https://scale.com/leaderboard/mask","lower_is_better":false,"metric":"percent","name":"MASK","paper":"https://arxiv.org/html/2503.03750v1","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":1,"score":95.33,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805-thinking","place":2,"score":94.2,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":3,"score":92,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":4,"score":89.27,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-opus-thinking","place":5,"score":87.87,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":6,"score":87.4,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":7,"score":86.46,"version":"GPT-OSS"},{"company":"openai","family":"o3","model":"o3-high","place":8,"score":84.47,"version":"o3"},{"company":"openai","family":"o3","model":"o3-medium","place":9,"score":82.6,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-2025-08","place":10,"score":82.6,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-pro-high","place":11,"score":82.5,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":12,"score":82.13,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":13,"score":80.28,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-2025-08-07","place":14,"score":79.33,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":15,"score":79,"version":"Claude 3"},{"company":"openai","family":"o4","model":"o4-mini-high","place":16,"score":78.6,"version":"o4"},{"company":"openai","family":"o4","model":"o4-mini-medium","place":17,"score":72.93,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":18,"score":72.33,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":19,"score":72.27,"version":"Claude 3.7"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":20,"score":71,"version":"Grok 4.1"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":21,"score":61.7,"version":"Gemini 3"},{"company":"openai","family":"o1","model":"o1-pro","place":22,"score":61.6,"version":"o1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":23,"score":61.46,"version":"GLM 4.5"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":24,"score":61.4,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":25,"score":61.4,"version":"GPT-4.1"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":26,"score":60.8,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":27,"score":60.07,"version":"GPT-4o"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":28,"score":59.27,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":29,"score":58,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":30,"score":57.32,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4.5-preview","place":31,"score":56.93,"version":"GPT-4.5"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":32,"score":56.5,"version":"Magistral 1.2"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b","place":33,"score":56.4,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-exp-03.25","place":34,"score":55.93,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":35,"score":55.67,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":36,"score":54.07,"version":"Llama 3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":37,"score":54,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-05-06","place":38,"score":53.07,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":39,"score":53,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":40,"score":51.93,"version":"Llama 3.3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":41,"score":51.13,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":42,"score":50.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":43,"score":50,"version":"GPT-4.1"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-low","place":44,"score":49.73,"version":"o3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":45,"score":49.73,"version":"Llama 4"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-thinking-exp-01-21","place":46,"score":49.53,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":47,"score":49.13,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":48,"score":49.07,"version":"Gemini 2.0"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-med","place":49,"score":48.93,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro-exp-02-05","place":50,"score":48.67,"version":"Gemini 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-large-2411","place":51,"score":47.53,"version":"Mistral Large"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":52,"score":47,"version":"DeepSeek V3.2"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31-high","place":53,"score":46.8,"version":"o3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":54,"score":46.67,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":55,"score":46.27,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":56,"score":44.53,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":57,"score":44.3,"version":"Gemini 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":58,"score":42.6,"version":"Mistral 3"},{"company":"xai","family":"Grok","model":"grok-4","place":59,"score":41.7,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":60,"score":17.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":61,"score":12.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":62,"score":12.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":63,"score":12,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":64,"score":5.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":65,"score":5.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":66,"score":3.9,"version":"Claude 4.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4 (t)","top_model_family":"Claude","top_model_id":"claude-sonnet-4-thinking","top_model_version":"Claude 4","top_open_source":0,"top_percent":95.33,"topic":"Safety / red teaming","total_models":1613,"url":"https://github.com/centerforaisafety/mask"},{"bench_models":5,"description":"Materials property prediction benchmark for scientific AI models.","github":"","has_recent":0,"id":"matbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MatBench","paper":"","placements":[{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":1,"score":72.8,"version":"Intern-S1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":64.9,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":60,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":53.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":49.7,"version":"Qwen3 VL"}],"stars":"","top_company":"internlm","top_model":"Intern-S1-Pro","top_model_family":"Intern","top_model_id":"intern-s1-pro","top_model_version":"Intern-S1","top_open_source":1,"top_percent":72.8,"topic":"Materials property prediction","total_models":1613,"url":""},{"bench_models":153,"description":"Competition-level mathematics across algebra, geometry, number theory, combinatorics.","github":"","has_recent":0,"id":"math","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MATH","paper":"https://arxiv.org/abs/2103.03874","placements":[{"company":"openai","family":"o3","model":"o3-mini","place":1,"score":97.9,"version":"o3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":2,"score":97,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":3,"score":96.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":96.7,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":5,"score":96.1,"version":"OLMo 3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":6,"score":96.1,"version":"Ling 2.0"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":7,"score":95.9,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":8,"score":95.6,"version":"OLMo 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":9,"score":95.44,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":10,"score":95.4,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":11,"score":95.07,"version":"Motif 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":12,"score":92.6,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":13,"score":91.8,"version":"Gemini 2.0"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":14,"score":91.4,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":15,"score":90.96,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":16,"score":90.9,"version":"Gemini 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":17,"score":89,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":18,"score":89,"version":"Gemma 3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":19,"score":88.56,"version":"Kanana 2"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":20,"score":87.4,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":21,"score":87.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":22,"score":86.5,"version":"Gemini 1.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":23,"score":86.26,"version":"Kanana 2"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":24,"score":83.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":25,"score":83.8,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":26,"score":83.2,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":27,"score":82.88,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":28,"score":80.2,"version":"Qwen2.5"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":29,"score":78.16,"version":"JoyAI-LLM"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":30,"score":77.92,"version":"Kanana 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":31,"score":77.9,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":32,"score":75.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":33,"score":74.3,"version":"Qwen2.5"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":34,"score":74,"version":"MobileLLM R1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":35,"score":73.8,"version":"Llama 3.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":36,"score":73.62,"version":"Motif 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":37,"score":73.5,"version":"LLaDA 2.0"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":38,"score":72.86,"version":"Seed 1.6"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":39,"score":72.5,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":40,"score":71.84,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":41,"score":71.8,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":42,"score":71.1,"version":"Gemma 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":43,"score":71,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":44,"score":70.2,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-instruct","place":45,"score":69.6,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":46,"score":69.42,"version":"Mistral 3.2"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":47,"score":69.3,"version":"Mistral 3.1"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":48,"score":68.8,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":49,"score":68.5,"version":"Qwen2.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":50,"score":68.3,"version":"Nemotron Nano v2"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":51,"score":68,"version":"Llama 3.1"},{"company":"mistral","family":"Ministral","model":"ministral-3-14b-base","place":52,"score":67.6,"version":"Ministral 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":53,"score":67.3,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":54,"score":66.6,"version":"Llama 3.1"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":55,"score":66.18,"version":"Seed 1.5"},{"company":"tencent","family":"WeDLM","model":"wedlm-8b-instruct","place":56,"score":64.8,"version":"WeDLM"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":57,"score":63.4,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":58,"score":63.32,"version":"Llama 4"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":59,"score":63,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":60,"score":62.62,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":61,"score":62.6,"version":"DeepSeek V3"},{"company":"mistral","family":"Ministral","model":"ministral-3-8b-base","place":62,"score":62.6,"version":"Ministral 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":63,"score":62.6,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":64,"score":62.58,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":65,"score":62.5,"version":"DeepSeek V3.2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":66,"score":62.3,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":67,"score":62.1,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":68,"score":62.02,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-base","place":69,"score":62,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":70,"score":61.62,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":71,"score":61.6,"version":"DeepSeek V3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-base","place":72,"score":61.2,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":73,"score":61.2,"version":"Llama 4"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":74,"score":61,"version":"GLM 4.5"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":75,"score":60.1,"version":"Ministral 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":76,"score":59.9,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":77,"score":59.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":78,"score":59.04,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":79,"score":58.5,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":80,"score":58.5,"version":"Qwen2.5"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":81,"score":58.4,"version":"Hermes 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-base","place":82,"score":57.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2-7b","place":83,"score":56.8,"version":"Qwen2"},{"company":"mistral","family":"Mathstral","model":"mathstral-7b","place":84,"score":56.6,"version":"Mathstral"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":85,"score":56.4,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":86,"score":55.6,"version":"Gemma 2"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":87,"score":55,"version":"Llama 3.1 Nemotron"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":88,"score":54.4,"version":"Kanana 2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":89,"score":53.7,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":90,"score":53.5,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":91,"score":51.9,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":92,"score":51.5,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":93,"score":51,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-base","place":94,"score":50.3,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-base","place":95,"score":50.3,"version":"Llama 4"},{"company":"zai","family":"GLM","model":"glm4-9b","place":96,"score":50.2,"version":"GLM 4"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":97,"score":50,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":98,"score":49.4,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":99,"score":49.2,"version":"OLMo 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":100,"score":48.86,"version":"Kanana 2"},{"company":"google","family":"Gemma","model":"gemma-3-12b-base","place":101,"score":48.7,"version":"Gemma 3"},{"company":"google","family":"Gemma 2","model":"gemma2-9b","place":102,"score":48.3,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":103,"score":48,"version":"Gemma 3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":104,"score":47.68,"version":"Kanana 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":105,"score":45.4,"version":"Gemma 3"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":106,"score":45.3,"version":"Moonlight"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":107,"score":44.4,"version":"DeepSeek Math"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":108,"score":43.7,"version":"Tulu 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":109,"score":43.4,"version":"OLMo 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":110,"score":43.3,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":111,"score":42.5,"version":"Llama 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":112,"score":42,"version":"Tulu 3"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":113,"score":41.9,"version":"Llama 3.0"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":114,"score":41.9,"version":"Hermes 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":115,"score":41.8,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":116,"score":41.6,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":117,"score":41.6,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-base","place":118,"score":40.5,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":119,"score":40,"version":"Ministral"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b","place":120,"score":37.5,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":121,"score":36.3,"version":"Qwen2.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":122,"score":31.5,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":123,"score":29.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b-base","place":124,"score":29.8,"version":"Qwen3"},{"company":"allenai","family":"Bolmo","model":"bolmo-7b","place":125,"score":29.8,"version":"Bolmo 7B"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":126,"score":29.8,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-4b-base","place":127,"score":29.4,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":128,"score":29.1,"version":"Llama 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":129,"score":28.4,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama3-8b","place":130,"score":28.4,"version":"Llama 3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":131,"score":27.8,"version":"K2 V2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":132,"score":27.2,"version":"Gemma 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m-base","place":133,"score":26.8,"version":"MobileLLM R1"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-360m","place":134,"score":26.6,"version":"MobileLLM R1"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":135,"score":25.8,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":136,"score":24.8,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":137,"score":24.3,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-0425-1b-instruct","place":138,"score":19.2,"version":"OLMo 2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":139,"score":19.2,"version":"SmolLM2"},{"company":"hku","family":"EvaByte","model":"evabyte-6.5b","place":140,"score":17.3,"version":"EvaByte 6.5B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":141,"score":15.3,"version":"Marin 32B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":142,"score":14.8,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":143,"score":13.8,"version":"Llama 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-360m-base","place":144,"score":13.4,"version":"MobileLLM R1"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":145,"score":13.1,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":146,"score":12.7,"version":"Mistral"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":147,"score":12.7,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":148,"score":11.8,"version":"Gemma 2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":149,"score":11.6,"version":"SmolLM2"},{"company":"alephalpha","family":"TFree","model":"tfree-hat-pretrained-7b-base","place":150,"score":10.9,"version":"TFree Hat"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":151,"score":10.4,"version":"Marin 32B"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":152,"score":8.4,"version":"Llama 1"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-140m","place":153,"score":7.4,"version":"MobileLLM R1"},{"company":"meta","family":"BLT","model":"blt-7b","place":154,"score":7.3,"version":"BLT 7B"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":155,"score":6.8,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":156,"score":6,"version":"Llama 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-0425-1b-base","place":157,"score":5.2,"version":"OLMo 2"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":158,"score":5.2,"version":"CodeLlama-7B"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":159,"score":5.1,"version":"Magpie"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-140m-base","place":160,"score":4.6,"version":"MobileLLM R1"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":161,"score":3.9,"version":"Llama 2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-instruct","place":162,"score":3.4,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":163,"score":3,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":164,"score":1.8,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":165,"score":1.6,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-1b","place":166,"score":0.6,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":167,"score":0.6,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":168,"score":0.4,"version":"SmolLM2"}],"stars":"1185","top_company":"openai","top_model":"o3 mini","top_model_family":"o3","top_model_id":"o3-mini","top_model_version":"o3","top_open_source":0,"top_percent":97.9,"topic":"Math (competition)","total_models":1613,"url":"https://github.com/hendrycks/math"},{"bench_models":4,"description":"Korean translation of the MATH competition benchmark.","github":"","has_recent":0,"id":"math-ko","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MATH-Ko","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":1,"score":58.2,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":2,"score":54.07,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":3,"score":47.42,"version":"Kanana 1.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":4,"score":45.58,"version":"Kanana 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b","top_model_version":"Qwen3","top_open_source":1,"top_percent":58.2,"topic":"Math (Korean)","total_models":1613,"url":""},{"bench_models":7,"description":"Level 5 subset of the MATH benchmark emphasizing the hardest competition-style problems.","github":"","has_recent":0,"id":"math-lvl5","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MATH Level 5","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":1,"score":73.62,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":2,"score":62.38,"version":"LFM2"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":3,"score":54.38,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":4,"score":52.18,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":5,"score":51.93,"version":"SmollM3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":6,"score":36.11,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":7,"score":24.06,"version":"Llama 3.2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-4B-Instruct-2507","top_model_family":"Qwen","top_model_id":"qwen3-4b-instruct-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":73.62,"topic":"Math (competition)","total_models":1613,"url":"https://github.com/hendrycks/math"},{"bench_models":135,"description":"500-problem slice of the MATH benchmark for challenging math reasoning.","github":"","has_recent":0,"id":"math500","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MATH500","paper":"","placements":[{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":1,"score":99.3,"version":"Motif 2"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":99.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":98.8,"version":"Gemini 2.5"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":4,"score":98.57,"version":"Cogito 671B v2.1"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":5,"score":98.2,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3","place":6,"score":98.1,"version":"o3"},{"company":"primeintellect","family":"INTELLECT","model":"intellect-3","place":7,"score":98.1,"version":"INTELLECT"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":8,"score":98,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":9,"score":98,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":10,"score":98,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":11,"score":98,"version":"Qwen3 Next"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":12,"score":97.8,"version":"GLM 4.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":13,"score":97.8,"version":"Nemotron Nano v2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":14,"score":97.4,"version":"Kimi K2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":15,"score":97.4,"version":"Falcon H1R"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":16,"score":97.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":17,"score":97.3,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":18,"score":97.2,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":19,"score":97.2,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":20,"score":97.2,"version":"Qwen3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":21,"score":97.1,"version":"JoyAI-LLM"},{"company":"zai","family":"GLM","model":"glm-4.5","place":22,"score":97,"version":"GLM 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":23,"score":96.8,"version":"MiniMax M1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":24,"score":96.8,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":25,"score":96.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":26,"score":96.8,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":27,"score":96.8,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":28,"score":96.8,"version":"Qwen3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":29,"score":96.8,"version":"LongCat-Flash-Lite"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":30,"score":96.7,"version":"Seed 1.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":31,"score":96.4,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":32,"score":96.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":33,"score":96.2,"version":"Qwen3"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":34,"score":96,"version":"MiniMax M1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":35,"score":96,"version":"GPT-OSS"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":36,"score":95.8,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":37,"score":95.6,"version":"Qwen3"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":38,"score":95.4,"version":"Llama 3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":39,"score":95.4,"version":"Gemini 2.5"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":40,"score":95.4,"version":"Phi-4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":41,"score":95.2,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-4-behemoth","place":42,"score":95,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":43,"score":95,"version":"Qwen3"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":44,"score":95,"version":"TeleChat3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":45,"score":94.8,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":46,"score":94.6,"version":"Kimi Linear"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":47,"score":94.4,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":48,"score":94.4,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":49,"score":94.2,"version":"Kimi Linear"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":50,"score":94,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":51,"score":94,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":52,"score":94,"version":"Claude 4"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":53,"score":93.7,"version":"Youtu-LLM"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":54,"score":92.8,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":55,"score":92.8,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":56,"score":92.4,"version":"GPT-4.1"},{"company":"arcee","family":"Trinity","model":"trinity-mini","place":57,"score":92.1,"version":"Trinity"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":58,"score":91.8,"version":"Gemini 2.0"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":59,"score":91.8,"version":"SmollM3"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":60,"score":91.6,"version":"Apriel Nemotron"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":61,"score":91.6,"version":"EXAONE 4.0"},{"company":"aquif","family":"Aquif","model":"aquif-3-moe-17b-a2.8b","place":62,"score":91.4,"version":"Aquif 3"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":63,"score":91.4,"version":"Nemotron-H"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":64,"score":91.2,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":65,"score":91,"version":"Mistral 3"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":66,"score":91,"version":"TeleChat3"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":67,"score":90.9,"version":"GLM 4.7"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":68,"score":90.85,"version":"Ouro"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":69,"score":90.8,"version":"QwQ"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":70,"score":90.8,"version":"DeepSeek R1"},{"company":"openai","family":"o1","model":"o1-mini","place":71,"score":90,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":72,"score":90,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":73,"score":90,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":74,"score":89.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":75,"score":89.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":76,"score":89.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":77,"score":89.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":78,"score":89.73,"version":"Qwen3"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":79,"score":89.6,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":80,"score":88.6,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":81,"score":88.47,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":82,"score":88.3,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":83,"score":88.3,"version":"Mistral 3.1"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":84,"score":87.96,"version":"LFM2.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-2509","place":85,"score":87.4,"version":"Magistral 1.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":86,"score":87.3,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":87,"score":86.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":88,"score":85.73,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":89,"score":85.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":90,"score":85,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":91,"score":84.8,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":92,"score":84.8,"version":"DeepSeek R1"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":93,"score":83.8,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":94,"score":83.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":95,"score":83.6,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":96,"score":83.2,"version":"Llama 4"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":97,"score":83.2,"version":"Gemma 3"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":98,"score":82.4,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":99,"score":82.2,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":100,"score":82,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":101,"score":81.92,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":102,"score":81.47,"version":"Mistral 3.2"},{"company":"microsoft","family":"Phi","model":"phi-4-14b","place":103,"score":80.8,"version":"Phi 4"},{"company":"liquidai","family":"LFM","model":"lfm2-350m-math","place":104,"score":79.4,"version":"LFM2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":105,"score":78.63,"version":"Nemotron 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":106,"score":77.8,"version":"Falcon H1"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":107,"score":77,"version":"JoyAI-LLM"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":108,"score":76.4,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":109,"score":75.8,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":110,"score":74.6,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-6b","place":111,"score":74.5,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":112,"score":74.2,"version":"Falcon H1"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":113,"score":74.2,"version":"LFM2"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":114,"score":74,"version":"Falcon H1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":115,"score":73.6,"version":"SmollM3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":116,"score":73.4,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":117,"score":73.2,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":118,"score":73,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":119,"score":70.6,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":120,"score":70.4,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":121,"score":69.02,"version":"Granite 3.3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":122,"score":69,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":123,"score":68.6,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":124,"score":68.6,"version":"Gemma 3"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":125,"score":65.2,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":126,"score":64.2,"version":"Qwen2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":127,"score":63.6,"version":"LFM2"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":128,"score":63.2,"version":"LFM2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":129,"score":62.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":130,"score":61.2,"version":"Qwen2.5"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m","place":131,"score":60.7,"version":"MobileLLM R1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":132,"score":59.6,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":133,"score":58.4,"version":"Falcon H1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":134,"score":58.2,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":135,"score":58.09,"version":"Granite 3.3"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":136,"score":58,"version":"Moonlight"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":137,"score":58,"version":"Qwen3"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":138,"score":54.47,"version":"EuroLLM"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":139,"score":54.2,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":140,"score":52.9,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":141,"score":52.8,"version":"Granite 3.2"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":142,"score":50.47,"version":"OLMo 2"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":143,"score":48.73,"version":"Granite 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":144,"score":48.4,"version":"Qwen2.5"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":145,"score":48.2,"version":"Trinity Large"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":146,"score":47.2,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":147,"score":46,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":148,"score":45.8,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":149,"score":45.4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":150,"score":45.2,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":151,"score":44.8,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":152,"score":42.6,"version":"Qwen2.5"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":153,"score":42.27,"version":"Apertus"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":154,"score":41.2,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":155,"score":41.2,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":156,"score":41,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":157,"score":40.8,"version":"Llama 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":158,"score":35.54,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":159,"score":35.07,"version":"Granite 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":160,"score":27.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":161,"score":25.8,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":162,"score":23.4,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":163,"score":19.8,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":164,"score":17.6,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":165,"score":13.2,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":166,"score":7.4,"version":"Llama 3.2"}],"stars":"","top_company":"motif","top_model":"Motif-2-12.7B-Reasoning","top_model_family":"Motif","top_model_id":"motif-2-12.7b-reasoning","top_model_version":"Motif 2","top_open_source":1,"top_percent":99.3,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Spanish MATH500 benchmark","github":"","has_recent":0,"id":"math500-es","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MATH500 (ES)","paper":"","placements":[{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":1,"score":88.8,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":2,"score":87.9,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":3,"score":87.5,"version":"SmolLM3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":4,"score":84.5,"version":"EXAONE Deep"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":5,"score":70.6,"version":"Qwen3"}],"stars":"","top_company":"lg","top_model":"EXAONE 4.0 1.2B","top_model_family":"EXAONE","top_model_id":"exaone-4.0-1.2b","top_model_version":"EXAONE 4.0","top_open_source":1,"top_percent":88.8,"topic":"Math (multilingual)","total_models":1613,"url":""},{"bench_models":12,"description":"Challenging math contest problems from MathArena Apex benchmark.","github":"","has_recent":0,"id":"matharena-apex","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MathArena Apex","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":1,"score":82.1,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":80.1,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":71.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":71.4,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":5,"score":52.6,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":47.4,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":7,"score":43.4,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":8,"score":31.1,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":26,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":23.4,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":11,"score":1.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":12,"score":1,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":0.5,"version":"Gemini 2.5"}],"stars":"","top_company":"bytedance","top_model":"Seed2.0 Pro","top_model_family":"Seed","top_model_id":"seed2.0-pro","top_model_version":"Seed 2.0","top_open_source":0,"top_percent":82.1,"topic":"Challenging Math Contest problems","total_models":1613,"url":""},{"bench_models":5,"description":"Visual math reasoning benchmark combining images and text across diverse mathematical tasks.","github":"","has_recent":0,"id":"mathverse","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MathVerse","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":82.9,"version":"Gemini 2.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":2,"score":79.9,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":3,"score":79.6,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":4,"score":78.8,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":5,"score":76,"version":"GLM 4.6V"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":82.9,"topic":"Math reasoning (multimodal)","total_models":1613,"url":""},{"bench_models":16,"description":"Compact MathVerse split focusing on single-image math puzzles and visual reasoning.","github":"","has_recent":0,"id":"mathverse-mini","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MathVerse-mini","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":1,"score":85,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":84.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":82.9,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":4,"score":76.8,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":5,"score":72.1,"version":"GLM 4.5V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":6,"score":70.6,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":7,"score":65.9,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":8,"score":65.9,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":9,"score":65.9,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":10,"score":62.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":60,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":12,"score":57.6,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":13,"score":52.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":14,"score":46.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":15,"score":36.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":16,"score":27,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-235B-A22B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-235b-a22b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":85,"topic":"Math reasoning (multimodal)","total_models":1613,"url":""},{"bench_models":13,"description":"Multi-image visual mathematical reasoning tasks from the MathVerse ecosystem.","github":"","has_recent":0,"id":"mathverse-vision","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MathVerse-Vision","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":84.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":82.9,"version":"Gemini 2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":3,"score":81,"version":"ERNIE 4.5 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":4,"score":72.1,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":5,"score":68.4,"version":"GLM 4.1V"},{"company":"tencent","family":"R","model":"r-4b-base","place":6,"score":64.9,"version":"R"},{"company":"stepfun","family":"Step","model":"step-3","place":7,"score":62.7,"version":"Step 3"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":8,"score":61.7,"version":"Keye-VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":9,"score":54.6,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":10,"score":41.2,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":11,"score":40.8,"version":"InternVL 3.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":12,"score":34,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl3-8b","place":13,"score":32.4,"version":"InternVL 3"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":84.1,"topic":"Math reasoning (multimodal)","total_models":1613,"url":""},{"bench_models":50,"description":"Visual math reasoning benchmark with problems that combine images (charts, diagrams) and text.","github":"https://github.com/mathllm/MATH-V","has_recent":0,"id":"mathvision","leaderboard":"https://mathllm.github.io/mathvision/#leaderboard","lower_is_better":false,"metric":"percent","name":"MathVision","paper":"https://proceedings.neurips.cc/paper_files/paper/2024/file/ad0edc7d5fa1a783f063646968b7315b-Paper-Datasets_and_Benchmarks_Track.pdf","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":88.6,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":86.6,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":86.2,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":86.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":5,"score":86,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":6,"score":84.2,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":84.2,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":8,"score":83.9,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":9,"score":83,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":83,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":11,"score":81.3,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":77.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":13,"score":77.1,"version":"Claude 4.5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":14,"score":75.95,"version":"Step3-VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":15,"score":74.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":16,"score":74.6,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":17,"score":74.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":18,"score":73.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":19,"score":73.3,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":20,"score":72.34,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":21,"score":72.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":22,"score":71.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":23,"score":71.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":24,"score":70.9,"version":"GPT-5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":25,"score":70.81,"version":"Step3-VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":26,"score":70.81,"version":"Step3-VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":27,"score":69.64,"version":"dots.vlm1"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":28,"score":68.77,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":29,"score":68.7,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":30,"score":68.7,"version":"Seed 1.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":31,"score":68,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":32,"score":65.7,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":33,"score":65.6,"version":"GLM 4.5V"},{"company":"stepfun","family":"Step","model":"step-3","place":34,"score":64.8,"version":"Step 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":35,"score":64.3,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":36,"score":63.5,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":37,"score":63.4,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":38,"score":62.3,"version":"GLM 4.6V"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":39,"score":61.9,"version":"ERNIE 4.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":40,"score":60.2,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":41,"score":59.65,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":42,"score":59.6,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":43,"score":54.4,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":44,"score":54.4,"version":"Kimi VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":45,"score":54.05,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":46,"score":53.9,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":47,"score":52.9,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":48,"score":52.7,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":49,"score":52.7,"version":"Claude 4"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":50,"score":52.05,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":51,"score":51.6,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":52,"score":46.6,"version":"GPT-5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":53,"score":39.8,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":54,"score":39.4,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":55,"score":38.1,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":56,"score":33.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":57,"score":31.6,"version":"Qwen3 VL"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":88.6,"topic":"Math reasoning (multimodal)","total_models":1613,"url":"https://mathllm.github.io/mathvision"},{"bench_models":68,"description":"Visual math reasoning across diverse tasks.","github":"","has_recent":0,"id":"mathvista","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MathVista","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":1,"score":90.1,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":89.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":89.1,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":87.7,"version":"Seed 1.8"},{"company":"openai","family":"o3","model":"o3","place":5,"score":86.8,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":86.1,"version":"Seed 1.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":7,"score":86,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":8,"score":85.8,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":9,"score":85.6,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":10,"score":85.6,"version":"Seed 1.5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":11,"score":85.5,"version":"Step3-VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":12,"score":85.2,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":13,"score":85.1,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":14,"score":85,"version":"dots.vlm1"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":15,"score":84.6,"version":"GLM 4.5V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":84.6,"version":"Gemini 2.5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":17,"score":83.97,"version":"Step3-VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":18,"score":83.97,"version":"Step3-VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":19,"score":83.88,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":20,"score":83.51,"version":"GLM 4.6V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":21,"score":83.5,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":22,"score":82.85,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":23,"score":82.8,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":24,"score":82.7,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":25,"score":82.7,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-5","place":26,"score":82.7,"version":"GPT-5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":27,"score":82.5,"version":"ERNIE 4.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":28,"score":81.9,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":29,"score":81.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":30,"score":81.2,"version":"Gemini 2.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":31,"score":81.2,"version":"Keye-VL 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":32,"score":80.9,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":33,"score":80.7,"version":"GLM 4.1V"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":34,"score":80.7,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":35,"score":80.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":36,"score":80.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":37,"score":80.2,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":38,"score":80.1,"version":"Kimi VL"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":39,"score":79.9,"version":"MiniCPM V 4.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":40,"score":79.86,"version":"MiMo VL"},{"company":"stepfun","family":"Step","model":"step-3","place":41,"score":79.2,"version":"Step 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":42,"score":79.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":43,"score":78.5,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":44,"score":78.4,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":45,"score":77.2,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":46,"score":77.1,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":47,"score":76.78,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":48,"score":74.6,"version":"Qwen2.5 VL"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":49,"score":73.7,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":50,"score":73.7,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":51,"score":73.7,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":52,"score":73.1,"version":"Claude 4.5"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":53,"score":70.7,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":54,"score":70.7,"version":"Llama 4"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":55,"score":69.4,"version":"Pixtral"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":56,"score":68.91,"version":"Mistral 3.1"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":57,"score":67.8,"version":"Eagle 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":58,"score":67.09,"version":"Mistral 3.2"},{"company":"openai","family":"GPT","model":"gpt-4o","place":59,"score":64.6,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":60,"score":64.4,"version":"Claude 3.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":61,"score":64.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":62,"score":62.5,"version":"Qwen2.5 VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":63,"score":62.2,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":64,"score":61.6,"version":"InternVL 3.5"},{"company":"unknown","family":"PLM","model":"plm-8b","place":65,"score":59.9,"version":"PLM"},{"company":"unknown","family":"PLM","model":"plm-3b","place":66,"score":59.1,"version":"PLM"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":67,"score":58.9,"version":"Molmo2"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":68,"score":58.6,"version":"Molmo"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":69,"score":58,"version":"Pixtral"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":70,"score":57.6,"version":"InternVL 3"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":71,"score":57.2,"version":"LLaVA OneVision"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-0827","place":72,"score":56.9,"version":"Gemini 1.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":73,"score":56.7,"version":"Molmo2"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":74,"score":54.2,"version":"Molmo2"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":75,"score":51.6,"version":"Molmo"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":76,"score":51.5,"version":"SmolVLM2"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":77,"score":44.8,"version":"Claude 3"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":78,"score":44.5,"version":"Molmo"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2.5 Thinking","top_model_family":"Kimi","top_model_id":"kimi-k2.5-thinking","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":90.1,"topic":"Multimodal math reasoning","total_models":1613,"url":"https://mathvista.github.io/"},{"bench_models":80,"description":"Lightweight subset of MathVista for quick evaluation of visual mathematical reasoning.","github":"","has_recent":0,"id":"mathvista-mini","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MathVista-Mini","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":90.3,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":2,"score":90.1,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":87.9,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":87.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":87.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":86.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":7,"score":85.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":8,"score":85.8,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":9,"score":85.6,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":10,"score":83.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":11,"score":83.1,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":82.7,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":13,"score":81.3,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking","place":14,"score":80.1,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":15,"score":80.1,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":16,"score":80,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":17,"score":79.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":18,"score":79.1,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":19,"score":77.2,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":20,"score":77.1,"version":"InternVL 3.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":21,"score":75.5,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":22,"score":74.8,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":23,"score":73.7,"version":"Qwen3 VL"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":24,"score":73.3,"version":"Manzano"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":25,"score":73.1,"version":"Bagel"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":26,"score":72.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":27,"score":72.4,"version":"Claude 4"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":28,"score":71.8,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":29,"score":70.5,"version":"Qwen2 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":30,"score":70.3,"version":"Gemini 2.5"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":31,"score":69.8,"version":"Manzano"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":32,"score":67.7,"version":"Claude 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":33,"score":67.7,"version":"InternVL 2.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":34,"score":67.5,"version":"LLaVA OneVision"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":35,"score":65.5,"version":"InternVL 2"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":36,"score":63.9,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":37,"score":63.8,"version":"GPT-4o"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":38,"score":63.2,"version":"LLaVA OneVision"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":39,"score":62.3,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":40,"score":61.3,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":41,"score":61.3,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":42,"score":60.5,"version":"InternVL 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":43,"score":59.6,"version":"GPT-5"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":44,"score":58.6,"version":"Molmo"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":45,"score":58.4,"version":"Gemini 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":46,"score":58.3,"version":"InternVL 2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":47,"score":58.2,"version":"Qwen2 VL"},{"company":"openai","family":"GPT","model":"gpt-4v","place":48,"score":58.1,"version":"GPT-4V"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":49,"score":58,"version":"Pixtral"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":50,"score":57.7,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":51,"score":57.3,"version":"Llama 3.2"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":52,"score":55.6,"version":"MM1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":53,"score":54.1,"version":"Qwen2.5 Omni"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":54,"score":53.2,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":55,"score":53.2,"version":"Cambrian"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":56,"score":51.6,"version":"Molmo"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":57,"score":51.5,"version":"Llama 3.2"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":58,"score":51.3,"version":"InternVL 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":59,"score":50.5,"version":"Claude 3"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":60,"score":49,"version":"Cambrian 1"},{"company":"baai","family":"Emu","model":"emu3-8b","place":61,"score":47.6,"version":"Emu3"},{"company":"bytedance","family":"LLaVA","model":"llava-next-34b","place":62,"score":46.5,"version":"LLaVA-NeXT"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":63,"score":46.4,"version":"Claude 3"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":64,"score":44.5,"version":"Molmo"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":65,"score":44.5,"version":"Phi-3"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":66,"score":44.4,"version":"MM1.5"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":67,"score":43.9,"version":"Phi 3.5"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":68,"score":42.5,"version":"Janus Pro"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":69,"score":40.9,"version":"GPT-5"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":70,"score":40.5,"version":"xGen-MM"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":71,"score":39.6,"version":"BLIP-3"},{"company":"apple","family":"MM1","model":"mm1-30b","place":72,"score":39.4,"version":"MM1"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":73,"score":38.7,"version":"MiniCPM V 2.0"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":74,"score":36.5,"version":"Janus Pro"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":75,"score":34,"version":"MolmoE"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":76,"score":30.6,"version":"Gemini Nano"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":77,"score":28.7,"version":"PaliGemma"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":78,"score":27.7,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":79,"score":25.6,"version":"LLaVA 1.5"},{"company":"baai","family":"Bunny","model":"bunny-4b","place":80,"score":0,"version":"Bunny"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":81,"score":0,"version":"BLIP-3O"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":90.3,"topic":"Math reasoning (multimodal)","total_models":1613,"url":""},{"bench_models":12,"description":"Multilingual instruction-following evaluation across English and multilingual original prompts.","github":"","has_recent":0,"id":"maxife","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MAXIFE","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":88.4,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":2,"score":88.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":3,"score":88,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":87.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":87.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":86.6,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":7,"score":85.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":8,"score":84,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":9,"score":83.7,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":10,"score":83.2,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":79.2,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":12,"score":72.8,"version":"Kimi K2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":88.4,"topic":"Instruction following (multilingual)","total_models":1613,"url":""},{"bench_models":201,"description":"Short Python problems with hidden tests.","github":"","has_recent":0,"id":"mbpp","leaderboard":"https://llm-stats.com/benchmarks","lower_is_better":false,"metric":"percent","name":"MBPP","paper":"https://arxiv.org/abs/2108.07732","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":1,"score":97.4,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":96.8,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":3,"score":96.2,"version":"Qwen3"},{"company":"openai","family":"o1","model":"o1-preview","place":4,"score":95.5,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":5,"score":95.5,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":95.2,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":7,"score":94.2,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":8,"score":94.16,"version":"Ministral 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":9,"score":93.4,"version":"DeepSeek V3.2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":10,"score":93.39,"version":"Nemotron Nano V2"},{"company":"openai","family":"o1","model":"o1-mini","place":11,"score":93.1,"version":"o1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":12,"score":92.9,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":13,"score":92.3,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":14,"score":92.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":15,"score":92.3,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":16,"score":91.8,"version":"Kimi K2"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":17,"score":91.8,"version":"IQuest Coder V1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":18,"score":91.6,"version":"OLMo 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":19,"score":91.05,"version":"MiniCPM 4.1"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":20,"score":91.05,"version":"Falcon H1R"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":21,"score":91,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":22,"score":90.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":23,"score":90.5,"version":"Qwen2.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":24,"score":90.5,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":25,"score":90.2,"version":"Qwen2.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":26,"score":89.7,"version":"Gemini 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":27,"score":89.4,"version":"DeepSeek Coder V2"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":28,"score":89.4,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":29,"score":89.4,"version":"Claude 3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":30,"score":89.4,"version":"KAT"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":31,"score":89.11,"version":"MiniCPM SALA"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":32,"score":88.9,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":33,"score":88.6,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":34,"score":88.29,"version":"LLaDA 2.0"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":35,"score":88.2,"version":"QwQ"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":36,"score":87.6,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":37,"score":87.6,"version":"DeepSeek V2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":38,"score":87.6,"version":"GPT-4o"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":39,"score":87.6,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":40,"score":86.65,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":41,"score":86.2,"version":"Seed Coder"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":42,"score":86,"version":"Grok 3"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":43,"score":85.8,"version":"Apriel Nemotron"},{"company":"openai","family":"GPT","model":"gpt-4-turbo-2024-04-09","place":44,"score":85.7,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":45,"score":85.4,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":46,"score":85.2,"version":"DeepSeek Coder V2"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":47,"score":85.2,"version":"KAT"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":48,"score":85.01,"version":"Ling 2.0"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":49,"score":85,"version":"Youtu-LLM"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":50,"score":84.8,"version":"K2 V2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":51,"score":84.7,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":52,"score":84.7,"version":"Qwen2.5"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":53,"score":84.6,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":54,"score":84,"version":"Granite 4.0"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":55,"score":83.8,"version":"MiniMax M2"},{"company":"anthropic","family":"claude-3-sonnet","model":"claude-3-sonnet-mar-2024","place":56,"score":83.6,"version":"claude-3-sonnet (Mar 2024)"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":57,"score":83.6,"version":"Seed 1.6"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":58,"score":82.8,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-nov-2023","place":59,"score":82.5,"version":"GPT-3.5"},{"company":"meta","family":"Llama3-70B-instruct","model":"llama3-70b-instruct","place":60,"score":82.3,"version":"Llama3-70B-instruct"},{"company":"map","family":"OpenCoder","model":"opencoder-8b-instruct","place":61,"score":82,"version":"OpenCoder"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":62,"score":81.5,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":63,"score":81.4,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":64,"score":81.4,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":65,"score":81.32,"version":"Qwen3"},{"company":"artigenz","family":"Artigenz-Coder-DS-6.7B","model":"artigenz-coder-ds-6.7b","place":66,"score":80.7,"version":"Artigenz Coder DS"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":67,"score":80.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":68,"score":80.4,"version":"DeepSeek Coder"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":69,"score":80.4,"version":"Ouro"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-33b","place":70,"score":80.2,"version":"OpenCodeInterpreter DS"},{"company":"anthropic","family":"claude-3-haiku","model":"claude-3-haiku-mar-2024","place":71,"score":80.2,"version":"claude-3-haiku (Mar 2024)"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":72,"score":80.2,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":73,"score":80,"version":"Qwen2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":74,"score":79.6,"version":"Kimi Dev"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b-chat","place":75,"score":79.4,"version":"CodeQwen1.5"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":76,"score":79.4,"version":"Magicoder-S-DS-6.7B"},{"company":"whiterabbit","family":"WhiteRabbitNeo-33B-v1","model":"whiterabbitneo-33b-v1","place":77,"score":79.4,"version":"WhiteRabbitNeo-33B-v1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":78,"score":79,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":79,"score":78.8,"version":"Qwen3"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":80,"score":78.2,"version":"Codestral"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":81,"score":78.2,"version":"Qwen3"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b-instruct-v0.1","place":82,"score":78,"version":"StarCoder2"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":83,"score":77.75,"version":"LLaDA 2.0"},{"company":"xwin","family":"XwinCoder-34B","model":"xwincoder-34b","place":84,"score":77,"version":"XwinCoder-34B"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":85,"score":76.8,"version":"EXAONE 4.0"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":86,"score":76.7,"version":"Llama 3"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-6.7b","place":87,"score":76.5,"version":"OpenCodeInterpreter DS"},{"company":"code","family":"code-millenials-34B","model":"code-millenials-34b","place":88,"score":76.2,"version":"code-millenials-34B"},{"company":"speechless","family":"speechless-coder-ds-6.7B","model":"speechless-coder-ds-6.7b","place":89,"score":75.9,"version":"speechless-coder-ds-6.7B"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":90,"score":75.8,"version":"K2 V2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":91,"score":75.49,"version":"Nemotron 3"},{"company":"google","family":"Gemini","model":"gemini-1.0-pro","place":92,"score":75.4,"version":"Gemini 1.0"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":93,"score":75.4,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":94,"score":75.4,"version":"Llama 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-7b-instruct-v1.5","place":95,"score":75.2,"version":"DeepSeek Coder"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-python-34b-v1.0","place":96,"score":75.1,"version":"WizardCoder V1.0"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":97,"score":75.1,"version":"CodeLlama-34B"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":98,"score":74.9,"version":"DeepSeek Coder"},{"company":"wavecoder","family":"WaveCoder-Ultra-6.7B","model":"wavecoder-ultra-6.7b","place":99,"score":74.9,"version":"WaveCoder-Ultra-6.7B"},{"company":"huggingface","family":"starchat2-15b-v0.1","model":"starchat2-15b-v0.1","place":100,"score":74.9,"version":"starchat2-15b-v0.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":101,"score":74.4,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":102,"score":74.4,"version":"Gemma 3"},{"company":"cohere","family":"Command","model":"command-r-plus","place":103,"score":74.3,"version":"Command"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":104,"score":74.2,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":105,"score":74.2,"version":"DeepSeek V3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":106,"score":74,"version":"Granite 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":107,"score":73.9,"version":"DeepSeek R1"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b-instruct-v0.1","place":108,"score":73.8,"version":"Mixtral v0.1"},{"company":"speechless","family":"speechless-codellama-34B-v2.0","model":"speechless-codellama-34b-v2.0","place":109,"score":73.8,"version":"speechless-codellama-34B-v2.0"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":110,"score":73.8,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":111,"score":73.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b","place":112,"score":73.5,"version":"CodeQwen1.5"},{"company":"bigcode","family":"speechless-starcoder2-15b","model":"speechless-starcoder2-15b","place":113,"score":73.5,"version":"speechless-starcoder2-15b"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":114,"score":73.5,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":115,"score":73.4,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":116,"score":73,"version":"Granite 4.0"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":117,"score":73,"version":"Ouro"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":118,"score":73,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":119,"score":72.8,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":120,"score":72.58,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen1.5-72b-chat","place":121,"score":72.5,"version":"Qwen1.5"},{"company":"mistral","family":"Codestral","model":"codestral-22b-v0.1","place":122,"score":72.5,"version":"Codestral v0.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-base","place":123,"score":72,"version":"DeepSeek Coder"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":124,"score":72,"version":"Granite 4.0"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":125,"score":71.2,"version":"Mixtral"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":126,"score":71.2,"version":"Gemini 3"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":127,"score":71,"version":"K2 V2"},{"company":"meta","family":"CodeLlama","model":"codellama-70b","place":128,"score":70.8,"version":"CodeLlama"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":129,"score":70.8,"version":"DeepSeek Math"},{"company":"magicoder","family":"Magicoder-S-CL-7B","model":"magicoder-s-cl-7b","place":130,"score":70.6,"version":"Magicoder-S-CL-7B"},{"company":"mistral","family":"dolphin-2.6-mixtral-8x7b","model":"dolphin-2.6-mixtral-8x7b","place":131,"score":70.6,"version":"dolphin-2.6-mixtral-8x7b"},{"company":"google","family":"Codegemma","model":"codegemma-7b-it","place":132,"score":70.4,"version":"Codegemma"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":133,"score":69.3,"version":"CodeLlama-34B"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":134,"score":69.2,"version":"Llama 3.0"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":135,"score":68.7,"version":"Dream"},{"company":"mistral","family":"Codestral","model":"codestral-mamba","place":136,"score":68.5,"version":"Codestral"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":137,"score":68.3,"version":"Dream 7B"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":138,"score":68.3,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":139,"score":68,"version":"Qwen3"},{"company":"google","family":"CodeGemma","model":"codegemma-1.1-7b","place":140,"score":67.7,"version":"CodeGemma"},{"company":"databricks","family":"databricks/dbrx-instruct","model":"databricks-dbrx-instruct","place":141,"score":67.2,"version":"databricks/dbrx-instruct"},{"company":"bigcode","family":"speechless-starcoder2-7b","model":"speechless-starcoder2-7b","place":142,"score":66.7,"version":"speechless-starcoder2-7b"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":143,"score":66.7,"version":"SmollM3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":144,"score":65.96,"version":"Kanana 1.5"},{"company":"microsoft","family":"Phi-3-mini-4k-instruct","model":"phi-3-mini-4k-instruct","place":145,"score":65.9,"version":"Phi-3-mini-4k-instruct"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":146,"score":65.6,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":147,"score":65.51,"version":"Llama 3.1"},{"company":"radicalnumerics","family":"RND1","model":"rnd1-base-0910","place":148,"score":65.4,"version":"RND1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-instruct","place":149,"score":65.3,"version":"DeepSeek Coder"},{"company":"google","family":"Codegemma","model":"codegemma-7b","place":150,"score":65.1,"version":"codegemma-7b"},{"company":"meta","family":"Llama3-8B-instruct","model":"llama3-8b-instruct","place":151,"score":64.6,"version":"Llama3-8B-instruct"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":152,"score":64.4,"version":"Llama 3.1"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-15b-v1.0","place":153,"score":64.3,"version":"WizardCoder V1.0"},{"company":"speechless","family":"speechless-coding-7B-16k-tora","model":"speechless-coding-7b-16k-tora","place":154,"score":64.2,"version":"speechless-coding-7B-16k-tora"},{"company":"microsoft","family":"phi-2-2.7B","model":"phi-2-2.7b","place":155,"score":64,"version":"phi-2-2.7B"},{"company":"openchat","family":"OpenChat-3.5-7B-0106","model":"openchat-3.5-7b-0106","place":156,"score":63.8,"version":"OpenChat-3.5-7B-0106"},{"company":"codellama","family":"CodeLlama-13B","model":"codellama-13b","place":157,"score":63.5,"version":"CodeLlama-13B"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":158,"score":63.08,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":159,"score":63,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":160,"score":62.4,"version":"Llama 3.1"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":161,"score":62.39,"version":"Kanana 2"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":162,"score":61.8,"version":"K2 V2"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":163,"score":61.59,"version":"Solar Pro"},{"company":"meta","family":"Llama3-8B-base","model":"llama3-8b-base","place":164,"score":61.4,"version":"Llama3-8B-base"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":165,"score":60.7,"version":"Mixtral"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":166,"score":60.6,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":167,"score":60.4,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":168,"score":60.3,"version":"Qwen2.5"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":169,"score":60.27,"version":"Phi 3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":170,"score":60.21,"version":"Kanana 2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":171,"score":60.2,"version":"OLMo 3"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":172,"score":59.5,"version":"CodeLlama-7B"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b-instruct-v0.1","place":173,"score":59.5,"version":"Mixtral v0.1"},{"company":"microsoft","family":"WizardCoder-Python-7B-V1.0","model":"wizardcoder-python-7b-v1.0","place":174,"score":58.5,"version":"WizardCoder-Python-7B-V1.0"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":175,"score":57.6,"version":"K2 V2"},{"company":"mistral","family":"MistralHermes","model":"mistralhermes-codepro-7b-v1","place":176,"score":57.4,"version":"MistralHermes CodePro v1"},{"company":"speechless","family":"speechless-code-mistral-7B-v1.0","model":"speechless-code-mistral-7b-v1.0","place":177,"score":57.4,"version":"speechless-code-mistral-7B-v1.0"},{"company":"google","family":"Gemma","model":"gemma-1.1-7b-it","place":178,"score":57.1,"version":"gemma-1.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-base","place":179,"score":56.9,"version":"DeepSeek Coder"},{"company":"salesforce","family":"CodeT5+-16B","model":"codet5-16b","place":180,"score":56.6,"version":"CodeT5+-16B"},{"company":"google","family":"Codegemma","model":"codegemma-2b","place":181,"score":55.6,"version":"codegemma-2b"},{"company":"bigcode","family":"StarCoder-15B","model":"starcoder-15b","place":182,"score":55.1,"version":"StarCoder"},{"company":"stable","family":"stable-code-3B","model":"stable-code-3b","place":183,"score":54.8,"version":"stable-code-3B"},{"company":"salesforce","family":"CodeGen","model":"codegen-16b","place":184,"score":54.2,"version":"CodeGen"},{"company":"salesforce","family":"CodeT5+-6B","model":"codet5-6b","place":185,"score":52.9,"version":"CodeT5+-6B"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":186,"score":52.6,"version":"gemma-7b"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":187,"score":52.5,"version":"CodeLlama-7B"},{"company":"openhermes","family":"OpenHermes-2.5-Code-290k-13B","model":"openhermes-2.5-code-290k-13b","place":188,"score":52.4,"version":"OpenHermes-2.5-Code-290k-13B"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":189,"score":52.2,"version":"Llama 3.1"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":190,"score":51.9,"version":"Mistral"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":191,"score":51.5,"version":"DeepSeek R1"},{"company":"salesforce","family":"CodeGen","model":"codegen-6b","place":192,"score":50.8,"version":"CodeGen"},{"company":"xdan","family":"xDAN-L1-Chat-RL-v1-7B","model":"xdan-l1-chat-rl-v1-7b","place":193,"score":50.3,"version":"xDAN-L1-Chat-RL-v1-7B"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":194,"score":50.3,"version":"Llama 3.2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":195,"score":50.2,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":196,"score":49.8,"version":"Llama 2"},{"company":"salesforce","family":"CodeT5+-2B","model":"codet5-2b","place":197,"score":48.4,"version":"CodeT5+-2B"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":198,"score":48.2,"version":"CodeLlama-7B"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":199,"score":47.5,"version":"Mistral"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-instruct","place":200,"score":47.2,"version":"CoDA"},{"company":"google","family":"Gemma","model":"gemma-7b-it","place":201,"score":47.1,"version":"Gemma"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":202,"score":46.8,"version":"MobileLLM"},{"company":"salesforce","family":"CodeGen","model":"codegen-2b","place":203,"score":46.3,"version":"CodeGen"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":204,"score":44.7,"version":"Mistral v0.2"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":205,"score":43.9,"version":"Solar Mini"},{"company":"huggingface","family":"Zephyr","model":"zephyr-beta-7b","place":206,"score":42.1,"version":"Zephyr β-7B"},{"company":"google","family":"Gemma","model":"gemma-2b","place":207,"score":41.8,"version":"gemma-2b"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":208,"score":40.9,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":209,"score":39.6,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":210,"score":35.7,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":211,"score":35.4,"version":"Llama 2"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-base","place":212,"score":35.2,"version":"CoDA"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":213,"score":35.2,"version":"Gemma 3"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":214,"score":31.5,"version":"LLaDA"},{"company":"google","family":"gemma-1.1-2b-it","model":"gemma-1.1-2b-it","place":215,"score":29.8,"version":"gemma-1.1"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":216,"score":26.1,"version":"Llama 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":217,"score":12.4,"version":"Gemma 3"},{"company":"zysecai","family":"Zyte","model":"zyte-1b","place":218,"score":9.8,"version":"Zyte"}],"stars":"36312","top_company":"moonshotai","top_model":"Kimi-K2 Thinking","top_model_family":"Kimi","top_model_id":"kimi-k2-thinking","top_model_version":"Kimi K2","top_open_source":1,"top_percent":97.4,"topic":"Code generation","total_models":1613,"url":"https://huggingface.co/datasets/Muennighoff/mbpp"},{"bench_models":4,"description":"Korean translation of MBPP code generation benchmark.","github":"","has_recent":0,"id":"mbpp-ko","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MBPP-Ko","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":1,"score":66.84,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":2,"score":65.41,"version":"Kanana 1.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":3,"score":61.55,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":4,"score":57.29,"version":"Kanana 2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b","top_model_version":"Qwen3","top_open_source":1,"top_percent":66.84,"topic":"Code generation (Korean)","total_models":1613,"url":""},{"bench_models":207,"description":"Extended MBPP with more tests and stricter evaluation.","github":"","has_recent":0,"id":"mbpp-plus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MBPP+","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.6","place":1,"score":94.2,"version":"GLM 4.6"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":2,"score":92.6,"version":"DeepSeek V3.1"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":3,"score":88.62,"version":"Trinity Large"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":4,"score":88.6,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":5,"score":86,"version":"Llama 3.1"},{"company":"skt","family":"A.X","model":"ax-k1","place":6,"score":85.7,"version":"A.X"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":83.9,"version":"Claude 4.5"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":8,"score":82.5,"version":"Llama 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":9,"score":82.3,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":10,"score":82.3,"version":"Claude 4.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":11,"score":81.8,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":12,"score":81.5,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":13,"score":81,"version":"ERNIE 4.5"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":14,"score":80.95,"version":"Trinity Large"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":15,"score":80.89,"version":"Ling 2.0"},{"company":"openai","family":"o1","model":"o1-preview","place":16,"score":80.2,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":17,"score":80.2,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":18,"score":79.9,"version":"GPT-4.1"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":19,"score":79.4,"version":"EXAONE 3.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":20,"score":79.37,"version":"LLaDA 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":21,"score":79.1,"version":"Gemini 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":22,"score":78.84,"version":"LLaDA 2.1"},{"company":"openai","family":"o1","model":"o1-mini","place":23,"score":78.8,"version":"o1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":24,"score":78.24,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":25,"score":78.21,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":26,"score":77.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":27,"score":77.8,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":28,"score":77.8,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":29,"score":77.6,"version":"Qwen3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":30,"score":77.5,"version":"Mi:dm 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":31,"score":77.25,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":32,"score":77.25,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":33,"score":77.2,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":34,"score":77.2,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":35,"score":77.2,"version":"DeepSeek V3.2"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":36,"score":77.2,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":37,"score":77,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":38,"score":77,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":39,"score":76.72,"version":"LLaDA 2.1"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":40,"score":76.2,"version":"KAT"},{"company":"essential","family":"Rnj","model":"rnj-1","place":41,"score":75.7,"version":"Rnj 1"},{"company":"zai","family":"GLM","model":"glm-4.7","place":42,"score":75.7,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":43,"score":75.6,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":44,"score":75.4,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":45,"score":75.13,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":46,"score":75.13,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-instruct","place":47,"score":75.1,"version":"DeepSeek Coder V2"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":48,"score":74.6,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":49,"score":74.6,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":50,"score":74.3,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v2.5","place":51,"score":74.1,"version":"DeepSeek V2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":52,"score":74.1,"version":"Kimi K2"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":53,"score":74.07,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":54,"score":74.07,"version":"Qwen2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":55,"score":74.07,"version":"LLaDA 2.1"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":56,"score":73.81,"version":"Llama 3.3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":57,"score":73.81,"version":"Kanana 2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":58,"score":73.8,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":59,"score":73.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":60,"score":73.3,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":61,"score":73.3,"version":"Claude 3"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":62,"score":73.3,"version":"Seed Coder"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":63,"score":73.02,"version":"Kanana 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":64,"score":73,"version":"DeepSeek V3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":65,"score":72.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":66,"score":72.8,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":67,"score":72.69,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":68,"score":72.4,"version":"Kimi Linear"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":69,"score":72.22,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":70,"score":72.2,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":71,"score":72.2,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":72,"score":72.2,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":73,"score":72.2,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":74,"score":71.96,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":75,"score":71.96,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":76,"score":71.7,"version":"Qwen2.5"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":77,"score":71.7,"version":"Youtu-LLM"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":78,"score":71.43,"version":"Falcon H1"},{"company":"map","family":"OpenCoder","model":"opencoder-8b-instruct","place":79,"score":71.4,"version":"OpenCoder"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":80,"score":71.4,"version":"MiMo V2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":81,"score":71.2,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":82,"score":71,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":83,"score":70.6,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":84,"score":70.6,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":85,"score":70.6,"version":"DeepSeek Coder V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-33b-instruct","place":86,"score":70.1,"version":"DeepSeek Coder"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":87,"score":70.1,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":88,"score":69.8,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-nov-2023","place":89,"score":69.7,"version":"GPT-3.5"},{"company":"artigenz","family":"Artigenz-Coder-DS-6.7B","model":"artigenz-coder-ds-6.7b","place":90,"score":69.6,"version":"Artigenz Coder DS"},{"company":"anthropic","family":"claude-3-sonnet","model":"claude-3-sonnet-mar-2024","place":91,"score":69.3,"version":"claude-3-sonnet (Mar 2024)"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":92,"score":69.3,"version":"KAT"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b-chat","place":93,"score":69,"version":"CodeQwen1.5"},{"company":"meta","family":"Llama3-70B-instruct","model":"llama3-70b-instruct","place":94,"score":69,"version":"Llama3-70B-instruct"},{"company":"magicoder","family":"Magicoder-S-DS-6.7B","model":"magicoder-s-ds-6.7b","place":95,"score":69,"version":"Magicoder-S-DS-6.7B"},{"company":"anthropic","family":"claude-3-haiku","model":"claude-3-haiku-mar-2024","place":96,"score":68.8,"version":"claude-3-haiku (Mar 2024)"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":97,"score":68.8,"version":"Kimi Dev"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":98,"score":68.78,"version":"Falcon H1"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-33b","place":99,"score":68.5,"version":"OpenCodeInterpreter DS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":100,"score":68.25,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":101,"score":68,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":102,"score":67.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":103,"score":67.7,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":104,"score":67.5,"version":"Gemini 1.5"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":105,"score":67.5,"version":"Codestral"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":106,"score":67.46,"version":"Falcon H1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":107,"score":67.2,"version":"OLMo 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":108,"score":66.93,"version":"Gemma 3"},{"company":"whiterabbit","family":"WhiteRabbitNeo-33B-v1","model":"whiterabbitneo-33b-v1","place":109,"score":66.9,"version":"WhiteRabbitNeo-33B-v1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":110,"score":66.7,"version":"OLMo 3"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":111,"score":66.6,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":112,"score":66.6,"version":"Qwen2.5"},{"company":"map","family":"OpenCodeInterpreter","model":"opencodeinterpreter-ds-6.7b","place":113,"score":66.4,"version":"OpenCodeInterpreter DS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":114,"score":66.2,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":115,"score":66.1,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":116,"score":66,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":117,"score":66,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":118,"score":65.9,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":119,"score":65.87,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":120,"score":65.7,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-instruct","place":121,"score":65.6,"version":"DeepSeek Coder"},{"company":"xai","family":"Grok","model":"grok-3-beta","place":122,"score":65.6,"version":"Grok 3"},{"company":"bigcode","family":"StarCoder","model":"starcoder2-15b-instruct-v0.1","place":123,"score":65.1,"version":"StarCoder2"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":124,"score":64.8,"version":"Llama 4"},{"company":"xwin","family":"XwinCoder-34B","model":"xwincoder-34b","place":125,"score":64.8,"version":"XwinCoder-34B"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":126,"score":64.8,"version":"Gemini 3"},{"company":"huggingface","family":"starchat2-15b-v0.1","model":"starchat2-15b-v0.1","place":127,"score":64.6,"version":"starchat2-15b-v0.1"},{"company":"code","family":"code-millenials-34B","model":"code-millenials-34b","place":128,"score":64.6,"version":"code-millenials-34B"},{"company":"speechless","family":"speechless-coder-ds-6.7B","model":"speechless-coder-ds-6.7b","place":129,"score":64.4,"version":"speechless-coder-ds-6.7B"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b-instruct-v0.1","place":130,"score":64.3,"version":"Mixtral v0.1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":131,"score":64.02,"version":"Falcon 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":132,"score":64,"version":"Granite 4.0"},{"company":"cohere","family":"Command","model":"command-r-plus","place":133,"score":63.5,"version":"Command"},{"company":"wavecoder","family":"WaveCoder-Ultra-6.7B","model":"wavecoder-ultra-6.7b","place":134,"score":63.5,"version":"WaveCoder-Ultra-6.7B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":135,"score":63.5,"version":"Qwen2.5"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-python-34b-v1.0","place":136,"score":63.2,"version":"WizardCoder V1.0"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-instruct","place":137,"score":63.2,"version":"CoDA"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":138,"score":62.7,"version":"Qwen3"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":139,"score":62.7,"version":"Ouro"},{"company":"bigcode","family":"speechless-starcoder2-15b","model":"speechless-starcoder2-15b","place":140,"score":62.4,"version":"speechless-starcoder2-15b"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":141,"score":62.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-7b-instruct-v1.5","place":142,"score":62.2,"version":"DeepSeek Coder"},{"company":"mistral","family":"Codestral","model":"codestral-22b-v0.1","place":143,"score":61.9,"version":"Codestral v0.1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-72b-chat","place":144,"score":61.6,"version":"Qwen1.5"},{"company":"google","family":"Gemini","model":"gemini-1.0-pro","place":145,"score":61.4,"version":"Gemini 1.0"},{"company":"speechless","family":"speechless-codellama-34B-v2.0","model":"speechless-codellama-34b-v2.0","place":146,"score":61.4,"version":"speechless-codellama-34B-v2.0"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":147,"score":61.2,"version":"Gemma 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":148,"score":61,"version":"DeepSeek R1"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-mini-instruct","place":149,"score":60.9,"version":"Mi:dm 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":150,"score":60.85,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"codeqwen1.5-7b","place":151,"score":60.8,"version":"CodeQwen1.5"},{"company":"magicoder","family":"Magicoder-S-CL-7B","model":"magicoder-s-cl-7b","place":152,"score":60.1,"version":"Magicoder-S-CL-7B"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b-instruct","place":153,"score":59.8,"version":"EXAONE 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":154,"score":59.79,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-large","place":155,"score":59.5,"version":"Mistral"},{"company":"mistral","family":"dolphin-2.6-mixtral-8x7b","model":"dolphin-2.6-mixtral-8x7b","place":156,"score":59,"version":"dolphin-2.6-mixtral-8x7b"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-6.7b-base","place":157,"score":58.7,"version":"DeepSeek Coder"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":158,"score":58.5,"version":"Qwen3"},{"company":"huawei","family":"Dream","model":"dream-7b-base","place":159,"score":57.4,"version":"Dream"},{"company":"google","family":"Codegemma","model":"codegemma-7b-it","place":160,"score":56.9,"version":"Codegemma"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":161,"score":56.7,"version":"SmollM3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":162,"score":56.61,"version":"Falcon H1"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":163,"score":56.5,"version":"OLMo 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":164,"score":56.35,"version":"Falcon H1"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":165,"score":56.3,"version":"CodeLlama-34B"},{"company":"bigcode","family":"speechless-starcoder2-7b","model":"speechless-starcoder2-7b","place":166,"score":56.3,"version":"speechless-starcoder2-7b"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":167,"score":56.3,"version":"Moonlight"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":168,"score":56.1,"version":"Dream 7B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":169,"score":56.08,"version":"Qwen2.5"},{"company":"databricks","family":"databricks/dbrx-instruct","model":"databricks-dbrx-instruct","place":170,"score":55.8,"version":"databricks/dbrx-instruct"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":171,"score":55.6,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":172,"score":55.29,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":173,"score":55.03,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-instruct","place":174,"score":54.8,"version":"DeepSeek Coder"},{"company":"meta","family":"Llama3-8B-instruct","model":"llama3-8b-instruct","place":175,"score":54.8,"version":"Llama3-8B-instruct"},{"company":"openchat","family":"OpenChat-3.5-7B-0106","model":"openchat-3.5-7b-0106","place":176,"score":54.5,"version":"OpenChat-3.5-7B-0106"},{"company":"microsoft","family":"WizardCoder","model":"wizardcoder-15b-v1.0","place":177,"score":54.2,"version":"WizardCoder V1.0"},{"company":"microsoft","family":"Phi-3-mini-4k-instruct","model":"phi-3-mini-4k-instruct","place":178,"score":54.2,"version":"Phi-3-mini-4k-instruct"},{"company":"microsoft","family":"phi-2-2.7B","model":"phi-2-2.7b","place":179,"score":54.2,"version":"phi-2-2.7B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":180,"score":54.2,"version":"Qwen2.5"},{"company":"codellama","family":"CodeLlama-13B","model":"codellama-13b","place":181,"score":52.6,"version":"CodeLlama-13B"},{"company":"google","family":"Codegemma","model":"codegemma-7b","place":182,"score":52.4,"version":"codegemma-7b"},{"company":"meta","family":"Llama3-8B-base","model":"llama3-8b-base","place":183,"score":51.6,"version":"Llama3-8B-base"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":184,"score":51.6,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":185,"score":51.1,"version":"Gemma 3"},{"company":"speechless","family":"speechless-coding-7B-16k-tora","model":"speechless-coding-7b-16k-tora","place":186,"score":50.6,"version":"speechless-coding-7B-16k-tora"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":187,"score":50.53,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":188,"score":50,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":189,"score":50,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":190,"score":49.74,"version":"Qwen3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b-instruct-v0.1","place":191,"score":49.7,"version":"Mixtral v0.1"},{"company":"microsoft","family":"WizardCoder-Python-7B-V1.0","model":"wizardcoder-python-7b-v1.0","place":192,"score":49.5,"version":"WizardCoder-Python-7B-V1.0"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":193,"score":49,"version":"OLMo 2"},{"company":"speechless","family":"speechless-code-mistral-7B-v1.0","model":"speechless-code-mistral-7b-v1.0","place":194,"score":48.7,"version":"speechless-code-mistral-7B-v1.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-1.3b-base","place":195,"score":47.9,"version":"DeepSeek Coder"},{"company":"salesforce","family":"CodeT5+-16B","model":"codet5-16b","place":196,"score":47.1,"version":"CodeT5+-16B"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":197,"score":47.08,"version":"Qwen3"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":198,"score":46.8,"version":"CodeLlama-7B"},{"company":"google","family":"Codegemma","model":"codegemma-2b","place":199,"score":46.6,"version":"codegemma-2b"},{"company":"mistral","family":"MistralHermes","model":"mistralhermes-codepro-7b-v1","place":200,"score":46.4,"version":"MistralHermes CodePro v1"},{"company":"bigcode","family":"StarCoder-15B","model":"starcoder-15b","place":201,"score":46.1,"version":"StarCoder"},{"company":"salesforce","family":"CoDA","model":"coda-1.7b-base","place":202,"score":46,"version":"CoDA"},{"company":"openhermes","family":"OpenHermes-2.5-Code-290k-13B","model":"openhermes-2.5-code-290k-13b","place":203,"score":45.8,"version":"OpenHermes-2.5-Code-290k-13B"},{"company":"stable","family":"stable-code-3B","model":"stable-code-3b","place":204,"score":45.8,"version":"stable-code-3B"},{"company":"salesforce","family":"CodeGen","model":"codegen-16b","place":205,"score":45.5,"version":"CodeGen"},{"company":"google","family":"Gemma","model":"gemma-1.1-7b-it","place":206,"score":45,"version":"gemma-1.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":207,"score":44.2,"version":"DeepSeek R1"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":208,"score":43.4,"version":"gemma-7b"},{"company":"salesforce","family":"CodeGen","model":"codegen-6b","place":209,"score":42.9,"version":"CodeGen"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":210,"score":42.1,"version":"Mistral"},{"company":"salesforce","family":"CodeT5+-6B","model":"codet5-6b","place":211,"score":41.5,"version":"CodeT5+-6B"},{"company":"xdan","family":"xDAN-L1-Chat-RL-v1-7B","model":"xdan-l1-chat-rl-v1-7b","place":212,"score":41.3,"version":"xDAN-L1-Chat-RL-v1-7B"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":213,"score":39.7,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":214,"score":39.68,"version":"Qwen2.5"},{"company":"salesforce","family":"CodeT5+-2B","model":"codet5-2b","place":215,"score":38.1,"version":"CodeT5+-2B"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0-2","place":216,"score":37,"version":"Mistral 0.2"},{"company":"google","family":"Gemma","model":"gemma-7b-it","place":217,"score":36.8,"version":"Gemma"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":218,"score":36.2,"version":"Solar Mini"},{"company":"salesforce","family":"CodeGen","model":"codegen-2b","place":219,"score":36,"version":"CodeGen"},{"company":"huggingface","family":"Zephyr","model":"zephyr-beta-7b","place":220,"score":34.7,"version":"Zephyr β-7B"},{"company":"google","family":"Gemma","model":"gemma-2b","place":221,"score":34.1,"version":"gemma-2b"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":222,"score":33.07,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":223,"score":29.37,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":224,"score":29.1,"version":"Llama 3.2"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":225,"score":28.6,"version":"LLaDA"},{"company":"google","family":"gemma-1.1-2b-it","model":"gemma-1.1-2b-it","place":226,"score":23.3,"version":"gemma-1.1"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":227,"score":17.2,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":228,"score":10.1,"version":"Gemma 3"},{"company":"zysecai","family":"Zyte","model":"zyte-1b","place":229,"score":7.5,"version":"Zyte"}],"stars":"","top_company":"zai","top_model":"GLM 4.6","top_model_family":"GLM","top_model_id":"glm-4.6","top_model_version":"GLM 4.6","top_open_source":1,"top_percent":94.2,"topic":"Code generation","total_models":1613,"url":""},{"bench_models":26,"description":"Aggregate MCP agent benchmark covering tool-use and planning tasks.","github":"","has_recent":0,"id":"mcp-atlas","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MCP-Atlas","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":69.2,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":62.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":62.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":4,"score":61.3,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":5,"score":60.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":60.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":60.6,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":8,"score":59.5,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":9,"score":59.5,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":10,"score":57.4,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":11,"score":55.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":54.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":54.1,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-2025-08-07","place":14,"score":44.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":15,"score":43.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":16,"score":43.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":17,"score":43.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":18,"score":43.8,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3-pro-2025-06-10-high","place":19,"score":43.6,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":20,"score":40.9,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":21,"score":40.9,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":22,"score":35.6,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":23,"score":34,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":24,"score":23.9,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":25,"score":12,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":26,"score":8.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":27,"score":8.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":28,"score":7.2,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":29,"score":3.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":30,"score":3.4,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":31,"score":0.8,"version":"Llama 4"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":69.2,"topic":"Agent evaluation","total_models":1613,"url":""},{"bench_models":31,"description":"Benchmarks multi-step tool-use agents across diverse task suites with a unified overall success metric.","github":"","has_recent":0,"id":"mcp-universe","leaderboard":"https://mcp-universe.github.io/","lower_is_better":false,"metric":"percent","name":"MCP Universe","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":50.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":47.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":3,"score":46.5,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":45.9,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":44.16,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-medium","place":6,"score":43.72,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":7,"score":35.6,"version":"Kimi K2"},{"company":"xai","family":"Grok","model":"grok-4","place":8,"score":33.33,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":9,"score":30.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":10,"score":29.44,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":11,"score":29.44,"version":"Claude 4"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":12,"score":29.4,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":13,"score":28.14,"version":"Claude 4"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":14,"score":27.27,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":15,"score":26.41,"version":"Grok Code"},{"company":"openai","family":"o3","model":"o3-mini-medium","place":16,"score":26.41,"version":"o3"},{"company":"openai","family":"o4","model":"o4-mini-medium","place":17,"score":25.97,"version":"o4"},{"company":"zai","family":"GLM","model":"glm-4.5","place":18,"score":24.68,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":19,"score":24.24,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":20,"score":22.94,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":21,"score":22.08,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":22,"score":22.08,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":23,"score":21.65,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":24,"score":21.65,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":25,"score":19.91,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":26,"score":19.48,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":27,"score":19.05,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":28,"score":18.18,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":29,"score":18.18,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":30,"score":18.18,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":31,"score":15.58,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":32,"score":14.29,"version":"DeepSeek V3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":50.7,"topic":"Agent evaluation","total_models":1613,"url":"https://mcp-universe.github.io/"},{"bench_models":21,"description":"Benchmark for Model Context Protocol (MCP) agent tool-use.","github":"https://github.com/eval-sys/mcpmark","has_recent":0,"id":"mcpmark","leaderboard":"https://mcpmark.ai/","lower_is_better":false,"metric":"percent","name":"MCPMark","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":57.5,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":53.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":50.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":46.9,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":5,"score":46.1,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":43.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":42.3,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":8,"score":38,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":9,"score":33.5,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":10,"score":33.3,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":11,"score":31.7,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":12,"score":29.9,"version":"Claude 4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":13,"score":29.5,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":14,"score":28.2,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3","place":15,"score":25.4,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-plus","place":16,"score":24.8,"version":"Qwen3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":17,"score":24.4,"version":"MiniMax M2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":18,"score":21.9,"version":"Kimi K2"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":19,"score":20.5,"version":"Grok Code"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":20,"score":20.4,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":21,"score":17.7,"version":"Qwen3"},{"company":"openai","family":"o4","model":"o4-mini","place":22,"score":17.3,"version":"o4"}],"stars":"127","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":57.5,"topic":"Agent tool-use (MCP)","total_models":1613,"url":"https://mcpmark.ai/"},{"bench_models":4,"description":"Multilingual variant of the Dolly instruction-following benchmark.","github":"","has_recent":0,"id":"mdolly","leaderboard":"","lower_is_better":false,"metric":"percent","name":"mDolly","paper":"","placements":[{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":1,"score":86.9,"version":"Aya"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":2,"score":77.5,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":3,"score":67.3,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":4,"score":61.6,"version":"Ministral 3"}],"stars":"","top_company":"cohere","top_model":"Tiny Aya Global","top_model_family":"Aya","top_model_id":"tiny-aya-global","top_model_version":"Aya","top_open_source":1,"top_percent":86.9,"topic":"Instruction following (multilingual)","total_models":1613,"url":""},{"bench_models":11,"description":"Multimodal medical expert question answering benchmark.","github":"","has_recent":0,"id":"medxpertqa-mm","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MedXpertQA-MM","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":76,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":73.3,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":70,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":67.3,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":5,"score":65.3,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":63.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":7,"score":62.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":8,"score":61.4,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":54,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":10,"score":47.6,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":11,"score":34.4,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":76,"topic":"Medical VQA","total_models":1613,"url":""},{"bench_models":33,"description":"METR evaluates AI agents on long-horizon coding and agentic tasks, measuring autonomous task completion time.","github":"","has_recent":0,"id":"metr","leaderboard":"","lower_is_better":false,"metric":"hours","name":"METR","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":4.82,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1-codex-max","place":2,"score":2.88,"version":"GPT-5.1 Codex"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":2.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":2.03,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":5,"score":1.9,"version":"Claude 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":6,"score":1.82,"version":"Grok 4"},{"company":"openai","family":"o3","model":"o3","place":7,"score":1.57,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":8,"score":1.43,"version":"Claude 4"},{"company":"openai","family":"o4","model":"o4-mini","place":9,"score":1.32,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":10,"score":1.25,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":11,"score":0.97,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":12,"score":0.93,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":13,"score":0.75,"version":"GPT-OSS"},{"company":"openai","family":"o1","model":"o1","place":14,"score":0.68,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview","place":15,"score":0.67,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":16,"score":0.53,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20241022","place":17,"score":0.5,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":18,"score":0.45,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":19,"score":0.38,"version":"DeepSeek V3"},{"company":"openai","family":"o1","model":"o1-preview","place":20,"score":0.37,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20240620","place":21,"score":0.32,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":22,"score":0.3,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4-1106-preview","place":23,"score":0.15,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":24,"score":0.15,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":25,"score":0.12,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":26,"score":0.1,"version":"Claude 3"},{"company":"openai","family":"GPT","model":"gpt-4","place":27,"score":0.08,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":28,"score":0.08,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4-0125-preview","place":29,"score":0.08,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen2-72b","place":30,"score":0.03,"version":"Qwen2"},{"company":"openai","family":"GPT","model":"gpt-3.5","place":31,"score":0.02,"version":"GPT-3"},{"company":"openai","family":"GPT","model":"gpt-2","place":32,"score":0,"version":"GPT-2"},{"company":"openai","family":"GPT","model":"gpt-3","place":33,"score":0,"version":"GPT-3"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":4.82,"topic":"Long task benchmark","total_models":1613,"url":"https://metr.org"},{"bench_models":6,"description":"Multi-page End-to-end Web Comprehension benchmark.","github":"","has_recent":0,"id":"mewc","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MEWC","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":89.8,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":82.1,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":78.7,"version":"Gemini 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":4,"score":74.4,"version":"MiniMax M2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":5,"score":55.6,"version":"MiniMax M2.1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":41.3,"version":"GPT-5.2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":89.8,"topic":"Web comprehension","total_models":1613,"url":""},{"bench_models":97,"description":"Multilingual grade school math word problems.","github":"","has_recent":0,"id":"mgsm","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MGSM","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805-thinking","place":1,"score":94.4,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":2,"score":94.2,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":3,"score":93.8,"version":"Claude 4"},{"company":"openai","family":"o4","model":"o4-mini","place":4,"score":93.4,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":5,"score":93,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-thinking","place":6,"score":93,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-5","place":7,"score":92.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":8,"score":92.6,"version":"Claude 3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":9,"score":92.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":10,"score":92.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":11,"score":92.4,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":12,"score":92.4,"version":"Llama 4"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":13,"score":92.4,"version":"Claude 3.7"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":14,"score":92.3,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":15,"score":92.3,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-base","place":16,"score":92.3,"version":"Llama 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":17,"score":92.3,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-exp-03.25","place":18,"score":92.2,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":19,"score":92.1,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":20,"score":92,"version":"Claude 3.5"},{"company":"openai","family":"o3","model":"o3-mini","place":21,"score":92,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":22,"score":92,"version":"GPT-OSS"},{"company":"openai","family":"o3","model":"o3","place":23,"score":91.7,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":24,"score":91.7,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":25,"score":91.69,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":26,"score":91.64,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":27,"score":91.6,"version":"Llama 3.1"},{"company":"xai","family":"Grok","model":"grok-3","place":28,"score":91.3,"version":"Grok 3"},{"company":"openai","family":"o3","model":"o3-mini","place":29,"score":91.3,"version":"o3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":30,"score":91.1,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3.3-instruct-turbo-70b","place":31,"score":91.1,"version":"Llama 3.3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":32,"score":90.9,"version":"Kimi K2"},{"company":"xai","family":"Grok","model":"grok-4","place":33,"score":90.9,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":34,"score":90.9,"version":"Claude 4"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3.1","place":35,"score":90.9,"version":"Mistral 3.1"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":36,"score":90.9,"version":"Grok 4"},{"company":"openai","family":"o1","model":"o1","place":37,"score":90.8,"version":"o1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":38,"score":90.8,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-3","place":39,"score":90.7,"version":"Claude 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":40,"score":90.7,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":41,"score":90.6,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":42,"score":90.6,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-base","place":43,"score":90.6,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":44,"score":90.53,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":45,"score":90.5,"version":"GPT-4o"},{"company":"xai","family":"Grok","model":"grok-3-mini-fast-high-reasoning","place":46,"score":90.4,"version":"Grok 3"},{"company":"xai","family":"Grok","model":"grok-3-mini-fast-low-reasoning","place":47,"score":90.4,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":48,"score":90.4,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20-thinking","place":49,"score":90,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":50,"score":89.8,"version":"Gemini 2.5"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":51,"score":89.58,"version":"Mistral 3.2"},{"company":"openai","family":"o1","model":"o1","place":52,"score":89.3,"version":"o1"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":53,"score":89.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":54,"score":89.2,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":55,"score":89,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":56,"score":89,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":57,"score":88.5,"version":"GPT-4"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":58,"score":88.44,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-4-scout","place":59,"score":88,"version":"Llama 1"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":60,"score":87.8,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":61,"score":87.7,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":62,"score":87.5,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":63,"score":87.28,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-large-2411","place":64,"score":87.2,"version":"Mistral Large"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":65,"score":87,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":66,"score":86.9,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.2-90b-vision","place":67,"score":86.9,"version":"Llama 3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":68,"score":86.6,"version":"Gemini 1.5"},{"company":"nvidia","family":"Nemotron","model":"llama-3.3-nemotron-super-thinking","place":69,"score":86.4,"version":"Nemotron Super"},{"company":"openai","family":"GPT","model":"gpt-4o-mini","place":70,"score":86.2,"version":"GPT-4o"},{"company":"xai","family":"Grok","model":"grok-2","place":71,"score":86.1,"version":"Grok 2"},{"company":"cohere","family":"Command","model":"command-a","place":72,"score":85.7,"version":"Command"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku","place":73,"score":85.6,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku-20241022","place":74,"score":84.6,"version":"Claude 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":75,"score":84.2,"version":"Mistral 3.1"},{"company":"mistral","family":"Mistral Small","model":"mistral-small-1.0","place":76,"score":84,"version":"Mistral Small"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":77,"score":83,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":78,"score":81.76,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":79,"score":79.18,"version":"OLMo 2"},{"company":"ai21","family":"Jamba","model":"jamba-1.5-large","place":80,"score":77.4,"version":"Jamba 1.5"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":81,"score":76.07,"version":"EuroLLM"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":82,"score":74.32,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":83,"score":73.68,"version":"Granite 4.0"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":84,"score":72.73,"version":"Apertus"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":85,"score":72.4,"version":"LFM2"},{"company":"ai21","family":"Jamba","model":"jamba-1.6-large","place":86,"score":71.2,"version":"Jamba 1.6"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":87,"score":69.3,"version":"GPT-4.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":88,"score":68.9,"version":"Llama 3.1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":89,"score":68.72,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":90,"score":66.56,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":91,"score":61.68,"version":"Llama 3.2"},{"company":"liquidai","family":"LFM","model":"lfm2-1.2b","place":92,"score":55.04,"version":"LFM2"},{"company":"liquidai","family":"LFM","model":"lfm2-700m","place":93,"score":45.36,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":94,"score":43.6,"version":"Gemma 3"},{"company":"ai21","family":"Jamba","model":"jamba-1.6-mini","place":95,"score":41.7,"version":"Jamba 1.6"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":96,"score":41.28,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":97,"score":38.72,"version":"Granite 4.0"},{"company":"ai21","family":"Jamba","model":"jamba-1.5-mini","place":98,"score":29.6,"version":"Jamba 1.5"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":99,"score":29.52,"version":"LFM2"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":100,"score":29.12,"version":"Llama 3.2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":101,"score":28.76,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":102,"score":28.64,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":103,"score":28.56,"version":"Granite 4.0"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.1 (2025-08-05) Thinking","top_model_family":"Claude","top_model_id":"claude-opus-4-1-20250805-thinking","top_model_version":"Claude 4.1","top_open_source":0,"top_percent":94.4,"topic":"Math (multilingual)","total_models":1613,"url":""},{"bench_models":19,"description":"Multimodal instruction-following benchmark evaluating accuracy on complex image-text tasks.","github":"","has_recent":0,"id":"miabench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MIABench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":96,"version":"Gemini 2.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":2,"score":95.1,"version":"Ovis 2.6"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":3,"score":94.8,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":4,"score":94.1,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":92.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":92.4,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":92.3,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":8,"score":92.3,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":9,"score":92.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":10,"score":91.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":91.6,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":12,"score":91.2,"version":"Claude 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":13,"score":91.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":14,"score":91.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":15,"score":90.7,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":16,"score":90.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":17,"score":89.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":18,"score":89.6,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":19,"score":89.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":20,"score":89.4,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":21,"score":83.6,"version":"Qwen3 VL"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":96,"topic":"Multimodal instruction following","total_models":1613,"url":""},{"bench_models":5,"description":"Visual question answering benchmark for biological microscopy images.","github":"","has_recent":0,"id":"microvqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MicroVQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":69,"version":"Gemini 3"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":2,"score":63.3,"version":"Intern-S1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":60.4,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":55.4,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":53.8,"version":"Qwen3 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":69,"topic":"Biological microscopy","total_models":1613,"url":""},{"bench_models":4,"description":"Needle-in-a-haystack multi-query benchmark at 128K context.","github":"","has_recent":0,"id":"mimo-niah-multi-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NIAH-Multi 128K","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":1,"score":99.5,"version":"Kimi K2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":2,"score":98.6,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":3,"score":97.2,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":4,"score":94.3,"version":"DeepSeek V3.2"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2 Base","top_model_family":"Kimi","top_model_id":"kimi-k2-base","top_model_version":"Kimi K2","top_open_source":1,"top_percent":99.5,"topic":"Long-context QA","total_models":1613,"url":""},{"bench_models":4,"description":"Needle-in-a-haystack multi-query benchmark at 32K context.","github":"","has_recent":0,"id":"mimo-niah-multi-32k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NIAH-Multi 32K","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":1,"score":99.8,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":2,"score":99.7,"version":"DeepSeek V3.1"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":3,"score":99.3,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":4,"score":85.6,"version":"DeepSeek V3.2"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2 Base","top_model_family":"Kimi","top_model_id":"kimi-k2-base","top_model_version":"Kimi K2","top_open_source":1,"top_percent":99.8,"topic":"Long-context QA","total_models":1613,"url":""},{"bench_models":4,"description":"Needle-in-a-haystack multi-query benchmark at 64K context.","github":"","has_recent":0,"id":"mimo-niah-multi-64k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NIAH-Multi 64K","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":1,"score":100,"version":"Kimi K2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":2,"score":99.9,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":3,"score":98.6,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":4,"score":85.9,"version":"DeepSeek V3.2"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2 Base","top_model_family":"Kimi","top_model_id":"kimi-k2-base","top_model_version":"Kimi K2","top_open_source":1,"top_percent":100,"topic":"Long-context QA","total_models":1613,"url":""},{"bench_models":15,"description":"Spatial navigation benchmark.","github":"","has_recent":0,"id":"mindcube","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MindCube","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-flash","place":1,"score":78.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":77.3,"version":"Gemini 3"},{"company":"xai","family":"Grok","model":"grok-4","place":3,"score":64.7,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":4,"score":62,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":61.7,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":61.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":7,"score":61.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":59.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":58.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":10,"score":53,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":11,"score":52.7,"version":"Grok 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":12,"score":50.4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":13,"score":49.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":14,"score":40.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":15,"score":38,"version":"GPT-4o"}],"stars":"","top_company":"google","top_model":"Gemini 3 Flash","top_model_family":"Gemini","top_model_id":"gemini-3-flash","top_model_version":"Gemini 3","top_open_source":0,"top_percent":78.3,"topic":"Spatial navigation","total_models":1613,"url":""},{"bench_models":17,"description":"Advanced quantitative reasoning set inspired by the Minerva benchmark for STEM problem solving.","github":"https://github.com/allenai/minerva-math","has_recent":0,"id":"minerva-math","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Minerva Math","paper":"https://arxiv.org/abs/2206.14858","placements":[{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":1,"score":98,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":2,"score":97.3,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":3,"score":96.9,"version":"OLMo 3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":4,"score":94.9,"version":"GLM 4.5"},{"company":"mbzuai","family":"K2","model":"k2-high-70b","place":5,"score":94.5,"version":"K2 V2"},{"company":"mbzuai","family":"K2","model":"k2-medium-70b","place":6,"score":90.6,"version":"K2 V2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":7,"score":85.3,"version":"MiniMax M2"},{"company":"mbzuai","family":"K2","model":"k2-low-70b","place":8,"score":85,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":9,"score":82.1,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":10,"score":74,"version":"Granite 4.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":67.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":65,"version":"Gemini 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":13,"score":64.06,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":14,"score":63.32,"version":"Granite 4.0"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":15,"score":62.4,"version":"Seed 1.8"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":16,"score":62.06,"version":"Granite 4.0"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":17,"score":49.9,"version":"Seed 1.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3 235B A22B Thinking","top_model_family":"Qwen","top_model_id":"qwen-3-235b-a22b-thinking","top_model_version":"Qwen3","top_open_source":1,"top_percent":98,"topic":"University-level math","total_models":1613,"url":"https://github.com/allenai/minerva-math"},{"bench_models":3,"description":"MiniF2F competition benchmark pass@1 accuracy.","github":"","has_recent":0,"id":"minif2f-pass1","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MiniF2F pass@1","paper":"","placements":[{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":1,"score":50,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":2,"score":12.1,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":3,"score":5.7,"version":"Qwen3"}],"stars":"","top_company":"nvidia","top_model":"NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","top_model_family":"Nemotron","top_model_id":"nvidia-nemotron-3-nano-30b-a3b-bf16","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":50,"topic":"Math competition","total_models":1613,"url":"https://github.com/deepmind/mathematics_dataset"},{"bench_models":3,"description":"MiniF2F competition benchmark pass@32 accuracy.","github":"","has_recent":0,"id":"minif2f-pass32","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MiniF2F pass@32","paper":"","placements":[{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":1,"score":79.9,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":2,"score":43,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":3,"score":16.8,"version":"Qwen3"}],"stars":"","top_company":"nvidia","top_model":"NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","top_model_family":"Nemotron","top_model_id":"nvidia-nemotron-3-nano-30b-a3b-bf16","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":79.9,"topic":"Math competition","total_models":1613,"url":"https://github.com/deepmind/mathematics_dataset"},{"bench_models":7,"description":"MiniF2F competition benchmark (test split).","github":"","has_recent":0,"id":"minif2f-test","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MiniF2F (Test)","paper":"","placements":[{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking","place":1,"score":81.6,"version":"LongCat-Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":2,"score":79.5,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":3,"score":51.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":41.8,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":5,"score":37.7,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":6,"score":27,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking-2507","place":7,"score":26.6,"version":"Qwen3"}],"stars":"","top_company":"meituan","top_model":"LongCat-Flash-Thinking","top_model_family":"LongCat","top_model_id":"longcat-flash-thinking","top_model_version":"LongCat-Flash","top_open_source":1,"top_percent":81.6,"topic":"Math competition","total_models":1613,"url":"https://github.com/deepmind/mathematics_dataset"},{"bench_models":9,"description":"Mixed-subject benchmark covering knowledge and reasoning tasks across domains.","github":"","has_recent":0,"id":"mixeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MixEval","paper":"","placements":[{"company":"openai","family":"o1","model":"o1-mini","place":1,"score":82.9,"version":"o1"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":2,"score":82.8,"version":"Apriel Nemotron"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":3,"score":80.6,"version":"EXAONE 4.0"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":4,"score":77.3,"version":"QwQ"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":5,"score":77.05,"version":"Qwen3"},{"company":"arcee","family":"AFM","model":"afm-4.5b","place":6,"score":73.75,"version":"AFM"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":7,"score":72.8,"version":"SmolLM3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":8,"score":64.3,"version":"Gemma 3"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":9,"score":62.1,"version":"Llama 3.1"}],"stars":"","top_company":"openai","top_model":"o1 Mini","top_model_family":"o1","top_model_id":"o1-mini","top_model_version":"o1","top_open_source":0,"top_percent":82.9,"topic":"Multi-task reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Hard subset of MixEval covering diverse reasoning tasks.","github":"","has_recent":0,"id":"mixeval-hard","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MixEval Hard","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":1,"score":31.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":2,"score":27.6,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":3,"score":26.9,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-3.1-3b","place":4,"score":24.9,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":5,"score":24.3,"version":"Qwen3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-4B","top_model_family":"Qwen","top_model_id":"qwen3-4b","top_model_version":"Qwen3","top_open_source":1,"top_percent":31.6,"topic":"Multi-task reasoning (hard)","total_models":1613,"url":""},{"bench_models":32,"description":"MLVU: Large-scale multi-task benchmark for video understanding.","github":"","has_recent":0,"id":"mlvu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MLVU","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":1,"score":87.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":2,"score":86.7,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":86.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":85.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":85.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":85.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":7,"score":85.6,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":8,"score":85,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":9,"score":83.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":10,"score":83.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":11,"score":83.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":83,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":13,"score":82.1,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":14,"score":82.1,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":15,"score":81.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":16,"score":81.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":17,"score":78.1,"version":"Qwen3 VL"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":18,"score":77.5,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":19,"score":75.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":20,"score":74.6,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":21,"score":73.5,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":22,"score":72.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":23,"score":71.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":24,"score":71.4,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":25,"score":69.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":26,"score":68.3,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":27,"score":68,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":28,"score":68,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":29,"score":61.6,"version":"GLM 4.6V"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":30,"score":55.2,"version":"SmolVLM2"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":31,"score":52.6,"version":"GPT-5"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-0.5b","place":32,"score":47.3,"version":"SmolVLM2"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-0.256b","place":33,"score":40.6,"version":"SmolVLM2"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-122B-A10B","top_model_family":"Qwen","top_model_id":"qwen3.5-122b-a10b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":87.3,"topic":"Large video understanding","total_models":1613,"url":""},{"bench_models":3,"description":"Multimodal browsing comprehension benchmark.","github":"","has_recent":0,"id":"mm-browsecomp","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MM-BrowseComp","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":46.3,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":25,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":7.2,"version":"Gemini 2.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":46.3,"topic":"Multimodal browsing","total_models":1613,"url":""},{"bench_models":9,"description":"Instruction-following benchmark assessing multimodal obedience to complex prompts.","github":"","has_recent":0,"id":"mm-ifeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MM-IFEval","paper":"https://arxiv.org/abs/2502.04688","placements":[{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":1,"score":52.29,"version":"LFM2.5-VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":2,"score":51.83,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":3,"score":47.31,"version":"InternVL 3.5"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":4,"score":46.35,"version":"LFM2-VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":5,"score":38.62,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":6,"score":38.49,"version":"InternVL 3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":7,"score":36.17,"version":"InternVL 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":8,"score":24.99,"version":"FastVLM"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":9,"score":19.42,"version":"SmolVLM2"}],"stars":"","top_company":"liquidai","top_model":"LFM2.5-VL-1.6B","top_model_family":"LFM","top_model_id":"lfm2.5-vl-1.6b","top_model_version":"LFM2.5-VL","top_open_source":1,"top_percent":52.29,"topic":"Multimodal instruction following","total_models":1613,"url":""},{"bench_models":22,"description":"Multi-turn multimodal instruction following benchmark evaluating dialogue quality and helpfulness.","github":"","has_recent":0,"id":"mm-mt-bench","leaderboard":"","lower_is_better":false,"metric":"score","name":"MM-MT-Bench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":1,"score":8.5,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":8.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":3,"score":8.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":4,"score":8.1,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":5,"score":7.8,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":6,"score":7.72,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":7,"score":7.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":8,"score":7.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":9,"score":7.6,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":10,"score":7.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":11,"score":7.5,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":12,"score":7.5,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":13,"score":7.5,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":14,"score":7.4,"version":"GPT-5"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":15,"score":7.4,"version":"Pixtral"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":16,"score":7.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":17,"score":6.2,"version":"GPT-5"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":18,"score":6.05,"version":"Pixtral"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-0827","place":19,"score":5.93,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":20,"score":5.9,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":21,"score":5.46,"version":"Claude 3"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":22,"score":4.95,"version":"LLaVA OneVision"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-235B-A22B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-235b-a22b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":8.5,"topic":"Multimodal instruction following","total_models":1613,"url":""},{"bench_models":25,"description":"MMBench v1.1 Chinese subset for evaluating multimodal LLMs.","github":"","has_recent":0,"id":"mmbench-v1.1-cn","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMBench v1.1 (CN)","paper":"https://arxiv.org/abs/2307.06281","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":91.3,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":90.6,"version":"Seed 1.8"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":3,"score":89.8,"version":"Keye-VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":4,"score":89.78,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":89.7,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":89.1,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":89,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":8,"score":88.3,"version":"GLM 4.5V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":88.3,"version":"Gemini 2.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":10,"score":88.24,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":11,"score":88.2,"version":"Qwen2.5 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":12,"score":88.2,"version":"GLM 4.6V"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":13,"score":87.7,"version":"Ovis 2.6"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":14,"score":86.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":15,"score":86.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":16,"score":84.9,"version":"Claude 4.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":17,"score":84.8,"version":"Gemma 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":18,"score":84.7,"version":"GLM 4.1V"},{"company":"openai","family":"GPT","model":"gpt-4o","place":19,"score":84.5,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":20,"score":84.5,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":21,"score":83.7,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":22,"score":83.4,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":23,"score":82.4,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":24,"score":82.2,"version":"InternVL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":25,"score":82,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":26,"score":81.9,"version":"MiMo VL"},{"company":"stepfun","family":"Step","model":"step-3","place":27,"score":81.5,"version":"Step 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":28,"score":80.7,"version":"Kimi VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":91.3,"topic":"Multimodal understanding (Chinese)","total_models":1613,"url":"https://mmbench.opencompass.org.cn/"},{"bench_models":34,"description":"MMBench v1.1 English subset for evaluating multimodal LLMs.","github":"","has_recent":0,"id":"mmbench-v1.1-en","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMBench v1.1 (EN)","paper":"https://arxiv.org/abs/2307.06281","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":93.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":93.19,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":3,"score":92.75,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":4,"score":92.7,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":5,"score":92.38,"version":"Step3-VL"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":6,"score":92.11,"version":"Seed 1.5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":7,"score":92.05,"version":"Step3-VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":8,"score":92.05,"version":"Step3-VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":9,"score":91.04,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":10,"score":90.55,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":90.1,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":12,"score":90.1,"version":"Seed 1.8"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":13,"score":89.91,"version":"MiMo VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":14,"score":89.9,"version":"Seed 1.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":15,"score":89.7,"version":"Keye-VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":89.55,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":17,"score":89.47,"version":"Seed 1.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":18,"score":89.32,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":19,"score":89.2,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":20,"score":88.9,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":21,"score":88.8,"version":"GLM 4.6V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":22,"score":88.3,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":23,"score":88.2,"version":"GLM 4.5V"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":24,"score":88.2,"version":"InternVL 3.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":25,"score":88.2,"version":"Ovis 2.6"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":26,"score":87.5,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":27,"score":86.9,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":28,"score":86.2,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":29,"score":85.8,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":30,"score":85.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":31,"score":84.8,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4o","place":32,"score":84.6,"version":"GPT-4o"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":33,"score":84.5,"version":"MiMo VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":34,"score":84.4,"version":"Kimi VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":35,"score":84.4,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":36,"score":83.5,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":37,"score":83.4,"version":"InternVL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":38,"score":81.6,"version":"Gemma 3"},{"company":"stepfun","family":"Step","model":"step-3","place":39,"score":81.1,"version":"Step 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":40,"score":80.1,"version":"Gemma 3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":93.3,"topic":"Multimodal understanding (English)","total_models":1613,"url":"https://mmbench.opencompass.org.cn/"},{"bench_models":52,"description":"English dev split of MMBench v1.1 measuring multimodal question answering.","github":"https://github.com/open-compass/MMBench","has_recent":0,"id":"mmbench-v1.1-en-dev","leaderboard":"https://mmbench.opencompass.org.cn/leaderboard","lower_is_better":false,"metric":"percent","name":"MMBench v1.1 (EN dev)","paper":"https://arxiv.org/abs/2307.06281","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":1,"score":94.2,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":93.7,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":93.7,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":92.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":5,"score":92.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":91.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":7,"score":90.6,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":90.1,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":9,"score":89.9,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":10,"score":89.7,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":89.2,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":12,"score":88.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":13,"score":88.6,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":88.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":15,"score":88.2,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":16,"score":87,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":17,"score":86.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":86.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":19,"score":85.9,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":20,"score":85.1,"version":"Qwen3 VL"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":21,"score":85,"version":"Bagel"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":22,"score":85,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":23,"score":84.9,"version":"Claude 4.1"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":24,"score":83.4,"version":"Manzano"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":25,"score":82.8,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":26,"score":82.4,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":27,"score":82.4,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":28,"score":82.4,"version":"Claude 4"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":29,"score":81.1,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":30,"score":80.41,"version":"Qwen2.5 VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":31,"score":79.81,"version":"LFM2-VL"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":32,"score":79.2,"version":"Janus Pro"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":33,"score":78.7,"version":"InternVL 2.5"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":34,"score":78.6,"version":"Phi-3"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":35,"score":78.6,"version":"BLIP-3O"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":36,"score":78.5,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":37,"score":78.18,"version":"InternVL 3.5"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":38,"score":78.1,"version":"Manzano"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":39,"score":77.8,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":40,"score":77.2,"version":"InternVL 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":41,"score":76.4,"version":"Qwen2.5 VL"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":42,"score":75.5,"version":"Janus Pro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":43,"score":74.8,"version":"Qwen2.5 Omni"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":44,"score":72.4,"version":"MM1.5"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":45,"score":69.6,"version":"MiniCPM V 2.0"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":46,"score":69.24,"version":"SmolVLM2"},{"company":"harmonai","family":"Harmon","model":"harmon-1.5b","place":47,"score":65.5,"version":"Harmon-1.5B"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":48,"score":61.7,"version":"VILA 1.5"},{"company":"baai","family":"Emu","model":"emu3-8b","place":49,"score":58.5,"version":"Emu3"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":50,"score":51.6,"version":"GPT-5"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":51,"score":0,"version":"BLIP-3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":52,"score":0,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":53,"score":0,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":54,"score":0,"version":"GPT-4o"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2.5","top_model_family":"Kimi","top_model_id":"kimi-k2.5","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":94.2,"topic":"General VQA","total_models":1613,"url":"https://mmbench.opencompass.org.cn/home"},{"bench_models":5,"description":"MME-CC multimodal evaluation suite.","github":"","has_recent":0,"id":"mme-cc","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MME-CC","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":56.9,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":43.4,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":42.7,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":4,"score":33.7,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":27.5,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":56.9,"topic":"Multimodal evaluation","total_models":1613,"url":""},{"bench_models":5,"description":"Elo-style scoring for the MME multimodal evaluation benchmark.","github":"","has_recent":0,"id":"mme-elo","leaderboard":"","lower_is_better":false,"metric":"points","name":"MME Elo","paper":"","placements":[{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":1,"score":2186.4,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":2,"score":2163.29,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":3,"score":2128.83,"version":"InternVL 3.5"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":4,"score":2050.9,"version":"LFM2-VL"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":5,"score":1792.5,"version":"SmolVLM2"}],"stars":"","top_company":"opengvlab","top_model":"InternVL3-2B","top_model_family":"Intern","top_model_id":"internvl3-2b","top_model_version":"InternVL 3","top_open_source":1,"top_percent":2186.4,"topic":"Multimodal perception","total_models":1613,"url":""},{"bench_models":7,"description":"MME-RealWorld Chinese split.","github":"","has_recent":0,"id":"mme-realworld-cn","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MME-RealWorld (cn)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-4o","place":1,"score":58.5,"version":"GPT-4o"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":2,"score":58.5,"version":"InternVL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":3,"score":55.5,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":4,"score":55,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":5,"score":51.2,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":6,"score":47.9,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":7,"score":40.6,"version":"Claude 3.7"}],"stars":"","top_company":"openai","top_model":"GPT-4o","top_model_family":"GPT","top_model_id":"gpt-4o","top_model_version":"GPT-4o","top_open_source":0,"top_percent":58.5,"topic":"Real-world perception (CN)","total_models":1613,"url":""},{"bench_models":7,"description":"MME-RealWorld English split.","github":"","has_recent":0,"id":"mme-realworld-en","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MME-RealWorld (en)","paper":"","placements":[{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":1,"score":59.1,"version":"MiMo VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":2,"score":57.5,"version":"GPT-4o"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":3,"score":57.4,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":4,"score":57.4,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":5,"score":56.1,"version":"InternVL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":6,"score":51.9,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":7,"score":50.8,"version":"Claude 3.7"}],"stars":"","top_company":"xiaomi","top_model":"MiMo-VL 7B-RL","top_model_family":"MiMo","top_model_id":"mimo-vl-7b-rl","top_model_version":"MiMo VL","top_open_source":1,"top_percent":59.1,"topic":"Real-world perception (EN)","total_models":1613,"url":""},{"bench_models":19,"description":"Multi-image understanding benchmark evaluating cross-image reasoning.","github":"","has_recent":0,"id":"mmiu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMIU","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":72.1,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":2,"score":71,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":68.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":4,"score":64.5,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":5,"score":62.4,"version":"GLM 4.1V"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":6,"score":61.2,"version":"Gemini 2.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":7,"score":55.5,"version":"Molmo2"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":8,"score":54.2,"version":"Molmo2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":9,"score":54.1,"version":"Claude 4.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":10,"score":51.7,"version":"Molmo2"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":11,"score":50.3,"version":"Keye-VL 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":12,"score":49.4,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":13,"score":49.2,"version":"InternVL 3.5"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":14,"score":48.4,"version":"Eagle 2.5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":15,"score":46.5,"version":"MiniCPM V 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":16,"score":43.2,"version":"Qwen3 VL"},{"company":"unknown","family":"PLM","model":"plm-3b","place":17,"score":40.6,"version":"PLM"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":18,"score":35.3,"version":"Qwen3 VL"},{"company":"unknown","family":"PLM","model":"plm-8b","place":19,"score":27.4,"version":"PLM"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":72.1,"topic":"Multi-image understanding","total_models":1613,"url":""},{"bench_models":4,"description":"MMLB-NIAH 128k long-context multimodal benchmark.","github":"","has_recent":0,"id":"mmlb-niah-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLB-NIAH (128k)","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":72.2,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":70.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":69.9,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":4,"score":62.4,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":72.2,"topic":"Multimodal long-context","total_models":1613,"url":""},{"bench_models":4,"description":"MMLB-VRAG 128k long-context multimodal benchmark.","github":"","has_recent":0,"id":"mmlb-vrag-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLB-VRAG (128k)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":88.9,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":83.2,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":79.6,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":4,"score":75.4,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":88.9,"topic":"Multimodal long-context","total_models":1613,"url":""},{"bench_models":2,"description":"128K-context variant of MMLongBench evaluating multimodal long-context understanding.","github":"","has_recent":0,"id":"mmlongbench-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLongBench-128K","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.6v","place":1,"score":64.1,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":2,"score":63.4,"version":"GLM 4.6V"}],"stars":"","top_company":"zai","top_model":"GLM-4.6V","top_model_family":"GLM","top_model_id":"glm-4.6v","top_model_version":"GLM 4.6V","top_open_source":1,"top_percent":64.1,"topic":"Long-context multimodal","total_models":1613,"url":""},{"bench_models":31,"description":"Evaluates long-context document understanding with mixed text, tables, and figures across multiple pages.","github":"","has_recent":0,"id":"mmlongbench-doc","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLongBench-Doc","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":61.9,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":2,"score":61.5,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":60.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":60.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":5,"score":59.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":6,"score":59,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":58.5,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":8,"score":56.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":9,"score":56.2,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":55.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":11,"score":55.4,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":12,"score":54.9,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":13,"score":54.5,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":14,"score":53,"version":"GLM 4.6V"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":15,"score":52.7,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":16,"score":52.6,"version":"Claude 4"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":17,"score":52,"version":"Ovis 2.6"},{"company":"openai","family":"GPT","model":"gpt-5","place":18,"score":51.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":19,"score":51.5,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":20,"score":50.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":21,"score":47.9,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":22,"score":44.7,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":23,"score":43.5,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":24,"score":42.4,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":25,"score":42.1,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":26,"score":42.1,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":27,"score":38.3,"version":"Gemini 2.5"},{"company":"stepfun","family":"Step","model":"step-3","place":28,"score":31.8,"version":"Step 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":29,"score":31.6,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":30,"score":28.4,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":31,"score":22.1,"version":"GPT-5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":61.9,"topic":"Long-context multimodal documents","total_models":1613,"url":""},{"bench_models":315,"description":"57 tasks spanning STEM, humanities, social sciences; broad knowledge and reasoning.","github":"https://github.com/hendrycks/test","has_recent":0,"id":"mmlu","leaderboard":"https://www.kaggle.com/benchmarks/open-benchmarks/mmlu","lower_is_better":false,"metric":"percent","name":"MMLU","paper":"https://arxiv.org/abs/2009.03300","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":93.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":93.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":93.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-2025-08-07","place":4,"score":93.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":5,"score":93.4,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1-20250805","place":6,"score":93.4,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":93.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":8,"score":92.9,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":92.9,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":10,"score":92.8,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":11,"score":92.8,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":92.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":13,"score":92.4,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":14,"score":92.3,"version":"o3"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":15,"score":92.3,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":16,"score":92.3,"version":"Seed 1.8"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":17,"score":91.9,"version":"o1"},{"company":"xai","family":"Grok","model":"grok-4","place":18,"score":91.6,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-0709","place":19,"score":91.6,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":20,"score":91.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":21,"score":91.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":22,"score":91.4,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":23,"score":90.4,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":24,"score":90.3,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":25,"score":90.3,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":26,"score":90.3,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":27,"score":90.3,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":28,"score":90.2,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet-20250219","place":29,"score":90.2,"version":"Claude 3.7"},{"company":"xai","family":"Grok","model":"grok-3","place":30,"score":90.1,"version":"Grok 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":31,"score":90.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":32,"score":90.1,"version":"Qwen3 VL"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":33,"score":90,"version":"o4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":34,"score":89.9,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":35,"score":89.5,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet-20241022","place":36,"score":89.5,"version":"Claude 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":37,"score":89.5,"version":"Kimi K2"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":38,"score":89.5,"version":"JoyAI-LLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":39,"score":89.4,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":40,"score":89.28,"version":"Qwen3 Next"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":41,"score":89.2,"version":"Grok 3"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":42,"score":88.83,"version":"Seed 1.6"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":43,"score":88.8,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":44,"score":88.6,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":45,"score":88.6,"version":"GPT-OSS"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":46,"score":88.35,"version":"Seed 1.5"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":47,"score":88.2,"version":"Solar Open"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":48,"score":88.1,"version":"GPT-4o"},{"company":"xai","family":"Grok","model":"grok-2-1212","place":49,"score":88,"version":"Grok 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":50,"score":88,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":51,"score":88,"version":"Llama 3.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":52,"score":87.98,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":53,"score":87.9,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":54,"score":87.9,"version":"GPT-4o"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":55,"score":87.9,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":56,"score":87.81,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":57,"score":87.8,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":58,"score":87.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":59,"score":87.8,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-2025-08-05","place":60,"score":87.7,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":61,"score":87.7,"version":"Gemini 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":62,"score":87.69,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":63,"score":87.6,"version":"Qwen3 VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":64,"score":87.4,"version":"DeepSeek V3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":65,"score":87.3,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":66,"score":87.26,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":67,"score":87.2,"version":"DeepSeek V3"},{"company":"arcee","family":"Trinity","model":"trinity-large-preview","place":68,"score":87.2,"version":"Trinity Large"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":69,"score":87.19,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":70,"score":87.13,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":71,"score":87,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":72,"score":87,"version":"Tulu 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":73,"score":86.9,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":74,"score":86.87,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":75,"score":86.7,"version":"MiMo V2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":76,"score":86.6,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":77,"score":86.5,"version":"DeepSeek V3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":78,"score":86.5,"version":"ERNIE 4.5"},{"company":"allenai","family":"OLMo","model":"olmo-3.1-think-32b","place":79,"score":86.4,"version":"OLMo 3.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":80,"score":86.11,"version":"Motif 2"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":81,"score":86.1,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":82,"score":86.1,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":83,"score":86.1,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":84,"score":86,"version":"Llama 3.1"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":85,"score":85.9,"version":"o3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":86,"score":85.6,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":87,"score":85.6,"version":"Qwen3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":88,"score":85.52,"version":"LongCat-Flash-Lite"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-base","place":89,"score":85.5,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":90,"score":85.5,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":91,"score":85.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":92,"score":85.5,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":93,"score":85.5,"version":"Llama 4"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":94,"score":85.4,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":95,"score":85.3,"version":"OLMo 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":96,"score":85.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":97,"score":85.2,"version":"Llama 3.1"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":98,"score":85.2,"version":"OLMo 3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":99,"score":85.16,"version":"Llama 4"},{"company":"mistral","family":"Mistral","model":"mistral-large-2411","place":100,"score":85.1,"version":"Mistral Large"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":101,"score":85.1,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":102,"score":85,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":103,"score":84.95,"version":"Qwen3"},{"company":"arcee","family":"Trinity","model":"trinity-mini","place":104,"score":84.95,"version":"Trinity"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":105,"score":84.9,"version":"Qwen3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":106,"score":84.9,"version":"Hermes 3"},{"company":"microsoft","family":"Phi","model":"phi-4-14b","place":107,"score":84.8,"version":"Phi 4"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":108,"score":84.7,"version":"JoyAI-LLM"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":109,"score":84.68,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":110,"score":84.62,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":111,"score":84.6,"version":"Qwen2.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":112,"score":84.4,"version":"Tulu 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":113,"score":84.07,"version":"Motif 2"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":114,"score":84.05,"version":"Falcon H1"},{"company":"openai","family":"o1","model":"o1-mini-2024-09-12","place":115,"score":84,"version":"o1"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":116,"score":84,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":117,"score":83.97,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":118,"score":83.8,"version":"Llama 3.1 Nemotron"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":119,"score":83.7,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":120,"score":83.61,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":121,"score":83.6,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":122,"score":83.55,"version":"GPT-OSS"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":123,"score":83.3,"version":"Tulu 3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":124,"score":83.3,"version":"GLM 4.5"},{"company":"aquif","family":"Aquif","model":"aquif-3-moe-17b-a2.8b","place":125,"score":83.2,"version":"Aquif 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":126,"score":83.15,"version":"LLaDA 2.0"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":127,"score":83.1,"version":"Tulu 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":128,"score":83,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":129,"score":83,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":130,"score":82.8,"version":"Qwen2.5"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":131,"score":82.75,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":132,"score":82.7,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":133,"score":82.6,"version":"Nemotron Nano v2"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":134,"score":82.58,"version":"Trinity Large"},{"company":"mistral","family":"Magistral","model":"magistral-small-2509","place":135,"score":82.3,"version":"Magistral 1.2"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-2025-08-05","place":136,"score":82.2,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":137,"score":82.12,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":138,"score":82.1,"version":"DeepSeek V3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":139,"score":82.09,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":140,"score":82.08,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":141,"score":82,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":142,"score":82,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":143,"score":81.8,"version":"Llama 3.3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":144,"score":81.61,"version":"Kanana 2"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":145,"score":81.5,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":146,"score":81.4,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":147,"score":81.4,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":148,"score":81.38,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":149,"score":81.14,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":150,"score":81.05,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3.5-haiku-20241022","place":151,"score":81,"version":"Claude 3.5"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":152,"score":80.9,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":153,"score":80.89,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":154,"score":80.8,"version":"Qwen2.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":155,"score":80.8,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":156,"score":80.7,"version":"Qwen3 VL"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":157,"score":80.62,"version":"Mistral 3.1"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":158,"score":80.53,"version":"GLM 4.7"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":159,"score":80.5,"version":"Mistral 3.1"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":160,"score":80.5,"version":"Mistral 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":161,"score":80.45,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":162,"score":80.4,"version":"Llama 4"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":163,"score":80.4,"version":"Hermes 3"},{"company":"openai","family":"GPT","model":"gpt-4.1-nano","place":164,"score":80.1,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":165,"score":79.91,"version":"Kimi Linear"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-base","place":166,"score":79.6,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-base","place":167,"score":79.6,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":168,"score":79.54,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":169,"score":79.5,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":170,"score":79.5,"version":"Qwen3"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":171,"score":79.5,"version":"Llama 3.0"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":172,"score":79.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":173,"score":79.3,"version":"Llama 3.1"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":174,"score":79.14,"version":"Solar Pro"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":175,"score":78.97,"version":"Qwen3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":176,"score":78.9,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":177,"score":78.7,"version":"Qwen3"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":178,"score":78.66,"version":"Phi 3.5 MoE"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":179,"score":78.6,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":180,"score":78.6,"version":"Gemma 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":181,"score":78.56,"version":"Nemotron 3"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":182,"score":78.45,"version":"Trinity Large"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":183,"score":78.44,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":184,"score":78.43,"version":"Llama 3.3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":185,"score":78.1,"version":"Motif 2"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":186,"score":78.02,"version":"Phi 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":187,"score":77.9,"version":"Nemotron Nano v2"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":188,"score":77.83,"version":"Phi 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":189,"score":77.75,"version":"Mixtral"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":190,"score":77.6,"version":"Qwen3"},{"company":"mistral","family":"Mixtral","model":"open-mixtral-8x22b-2404","place":191,"score":77.5,"version":"Mixtral"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":192,"score":77.32,"version":"Mistral 3.2"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":193,"score":77.3,"version":"Llama TFree HAT"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":194,"score":77.2,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-3-12b-it","place":195,"score":77.1,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":196,"score":77.1,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":197,"score":76.9,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":198,"score":76.83,"version":"Falcon H1"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":199,"score":76.76,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":200,"score":76.63,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":201,"score":76.6,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":202,"score":76.13,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":203,"score":76.1,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":204,"score":75.95,"version":"OLMo 2"},{"company":"cohere","family":"Command","model":"command-r-plus-08-2024","place":205,"score":75.7,"version":"Command"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":206,"score":75.44,"version":"Kanana 2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":207,"score":75.3,"version":"Gemma 3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":208,"score":75.2,"version":"K2 V2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":209,"score":75.2,"version":"OLMo 3"},{"company":"tencent","family":"WeDLM","model":"wedlm-8b-instruct","place":210,"score":75.14,"version":"WeDLM"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-001","place":211,"score":75,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":212,"score":74.99,"version":"Qwen1.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":213,"score":74.83,"version":"Kanana 2"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":214,"score":74.8,"version":"Mistral 3.2"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":215,"score":74.7,"version":"Marin 32B"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":216,"score":74.6,"version":"Ouro"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":217,"score":74.6,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":218,"score":74.6,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":219,"score":74.58,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":220,"score":74.5,"version":"Gemma 3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":221,"score":74.3,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":222,"score":74.2,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":223,"score":74.01,"version":"Falcon 3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":224,"score":73.7,"version":"Mi:dm 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":225,"score":73.64,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":226,"score":73.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":227,"score":73.19,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":228,"score":73.02,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":229,"score":73,"version":"Llama 3.1"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":230,"score":72.9,"version":"Marin 32B"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-nano-1b","place":231,"score":72.9,"version":"Aquif 3.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":232,"score":72.5,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":233,"score":72.25,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":234,"score":72.14,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":235,"score":71.9,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":236,"score":71.9,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":237,"score":71.56,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-instruct","place":238,"score":71.52,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":239,"score":71.2,"version":"Llama 3.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0-instruct","place":240,"score":71.11,"version":"Bielik v3.0"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":241,"score":70.81,"version":"Falcon 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":242,"score":70.7,"version":"Llama 3.1"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":243,"score":70.63,"version":"Mixtral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":244,"score":70.6,"version":"Mixtral"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":245,"score":70.4,"version":"Qwen3"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-2.7b","place":246,"score":70.2,"version":"Aquif 3.5"},{"company":"swissai","family":"Apertus","model":"apertus-70b-instruct","place":247,"score":70.2,"version":"Apertus"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":248,"score":69.9,"version":"Llama 2"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":249,"score":69.81,"version":"EuroLLM"},{"company":"radicalnumerics","family":"RND1","model":"rnd1-base-0910","place":250,"score":69.6,"version":"RND1"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":251,"score":69.5,"version":"Dream 7B"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":252,"score":69.4,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b","place":253,"score":69.36,"version":"Qwen1.5"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":254,"score":69,"version":"EXAONE 3.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":255,"score":68.7,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":256,"score":68.67,"version":"Llama 3.1"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":257,"score":68.65,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b-chat","place":258,"score":68.57,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":259,"score":68.5,"version":"Llama 3"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":260,"score":68.5,"version":"Ministral"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":261,"score":68.3,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":262,"score":68.25,"version":"Llama 3.1"},{"company":"cohere","family":"Command","model":"command-r-08-2024","place":263,"score":68.2,"version":"Command"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":264,"score":68.2,"version":"Tulu 3"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":265,"score":67.85,"version":"Apertus"},{"company":"qwen","family":"Qwen","model":"qwen-14b","place":266,"score":67.7,"version":"Qwen"},{"company":"mistral","family":"Codestral","model":"codestral-2501","place":267,"score":67.6,"version":"Codestral"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0","place":268,"score":67.55,"version":"Bielik v3.0"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.6-instruct","place":269,"score":67.53,"version":"Bielik v2.6"},{"company":"aquif","family":"Aquif","model":"aquif-3-3.2b","place":270,"score":67.5,"version":"Aquif 3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.5-instruct","place":271,"score":67.44,"version":"Bielik v2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":272,"score":67.43,"version":"Granite 4.0"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":273,"score":67.35,"version":"Ouro"},{"company":"microsoft","family":"Phi","model":"phi-4-mini-3.8b","place":274,"score":67.3,"version":"Phi 4 mini"},{"company":"meta","family":"Llama","model":"meta-llama-3-8b-instruct","place":275,"score":67.07,"version":"Llama 3"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":276,"score":66.79,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":277,"score":66.79,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":278,"score":66.77,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":279,"score":66.7,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":280,"score":66.7,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":281,"score":66.7,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":282,"score":66.7,"version":"Llama 3"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-1106","place":283,"score":66.4,"version":"GPT-3.5"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":284,"score":66.21,"version":"Solar Mini"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":285,"score":66.11,"version":"Falcon H1"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":286,"score":66.03,"version":"gemma-7b"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":287,"score":65.98,"version":"Granite 4.0"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":288,"score":65.9,"version":"LLaDA"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":289,"score":65.9,"version":"Tulu 3"},{"company":"arcee","family":"AFM","model":"afm-4.5b","place":290,"score":65.88,"version":"AFM"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":291,"score":65.62,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":292,"score":65.54,"version":"Granite 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-tfree-hat-pretrained-7b-dpo","place":293,"score":65.4,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":294,"score":65.3,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":295,"score":65.09,"version":"Qwen2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":296,"score":64.84,"version":"LFM2"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5-0106-gemma","place":297,"score":64.69,"version":"OpenChat 3.5 0106"},{"company":"servicenow","family":"Apriel","model":"apriel-5b-instruct","place":298,"score":64.6,"version":"Apriel"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.3-instruct","place":299,"score":64.57,"version":"Bielik v2.3"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":300,"score":64.42,"version":"LFM2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.2-instruct","place":301,"score":64.34,"version":"Bielik v2.2"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":302,"score":64.3,"version":"Gemma 2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.1-instruct","place":303,"score":64.18,"version":"Bielik v2.1"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.1","place":304,"score":64.16,"version":"Mistral v0.1"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":305,"score":64.1,"version":"EuroLLM"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.0-instruct","place":306,"score":63.71,"version":"Bielik v2.0"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3-instruct","place":307,"score":63.66,"version":"Bielik v3"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.2","place":308,"score":63.62,"version":"Mistral v0.2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2","place":309,"score":63.06,"version":"Bielik v2"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":310,"score":62.79,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":311,"score":62.7,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":312,"score":62.5,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":313,"score":62.47,"version":"Mistral"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":314,"score":62.46,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":315,"score":62.03,"version":"Falcon H1"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":316,"score":62,"version":"Magpie"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":317,"score":61.74,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-7b-chat","place":318,"score":61.65,"version":"Qwen1.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3","place":319,"score":61.32,"version":"Bielik v3"},{"company":"nvidia","family":"Nemotron Flash","model":"nemotron-flash-3b","place":320,"score":61.19,"version":"Nemotron Flash"},{"company":"nvidia","family":"Nemotron Flash","model":"nemotron-flash-3b-tp","place":321,"score":61.19,"version":"Nemotron Flash"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":322,"score":60.99,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":323,"score":60.78,"version":"Mistral v0.2"},{"company":"aquif","family":"Aquif","model":"aquif-3.5-a0.6b","place":324,"score":60.5,"version":"Aquif 3.5"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b","place":325,"score":60.4,"version":"EXAONE 3.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":326,"score":60.4,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":327,"score":60.35,"version":"Llama 3.2"},{"company":"nvidia","family":"Nemotron Flash","model":"nemotron-flash-3b","place":328,"score":60.34,"version":"Nemotron Flash"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":329,"score":60.33,"version":"Apertus"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":330,"score":60.1,"version":"Mistral"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":331,"score":59.84,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":332,"score":59.76,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":333,"score":59.69,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":334,"score":59.63,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":335,"score":59.6,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":336,"score":59.53,"version":"SmollM3"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b-instruct","place":337,"score":59.5,"version":"EXAONE 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":338,"score":59.11,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":339,"score":58.37,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":340,"score":58.35,"version":"Gemma 3"},{"company":"mistral","family":"Ministral","model":"ministral-3b-2410","place":341,"score":57.6,"version":"Ministral"},{"company":"mistral","family":"Ministral","model":"ministral-8b-2410","place":342,"score":57.3,"version":"Ministral"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":343,"score":57.18,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":344,"score":57.11,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":345,"score":56.8,"version":"Llama 1"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":346,"score":56.76,"version":"Falcon 3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-mini-instruct","place":347,"score":56.5,"version":"Mi:dm 2.0"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":348,"score":55.88,"version":"Granite 3.3"},{"company":"aquif","family":"Aquif","model":"aquif-3-0.4b","place":349,"score":55.6,"version":"Aquif 3"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":350,"score":55.6,"version":"Llama 2"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.1","place":351,"score":55.38,"version":"Mistral v0.1"},{"company":"liquidai","family":"LFM","model":"lfm2-1.2b","place":352,"score":55.23,"version":"LFM2"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":353,"score":53.4,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b-base","place":354,"score":52.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":355,"score":50.72,"version":"DeepSeek R1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":356,"score":50,"version":"SmolLM2"},{"company":"liquidai","family":"LFM","model":"lfm2-700m","place":357,"score":49.9,"version":"LFM2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-instruct-v0.1","place":358,"score":49.47,"version":"Bielik v0.1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":359,"score":49.3,"version":"Llama 3.2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-950m-base","place":360,"score":47.4,"version":"MobileLLM R1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":361,"score":46.6,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":362,"score":46.1,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":363,"score":46.07,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":364,"score":45.8,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":365,"score":45.46,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":366,"score":44.97,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":367,"score":44.93,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":368,"score":44.9,"version":"Qwen3"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":369,"score":44.8,"version":"MobileLLM"},{"company":"nvidia","family":"Nemotron Flash","model":"nemotron-flash-1b","place":370,"score":44.63,"version":"Nemotron Flash"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":371,"score":44.4,"version":"Llama 2"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":372,"score":44,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":373,"score":44,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":374,"score":43.43,"version":"LFM2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-v0.1","place":375,"score":43.2,"version":"Bielik v0.1"},{"company":"allenai","family":"OLMo","model":"olmo-2-0425-1b-base","place":376,"score":42.4,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":377,"score":42.3,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":378,"score":40.08,"version":"Gemma 3"},{"company":"pleias","family":"Baguettotron","model":"baguettotron-321m","place":379,"score":40,"version":"Baguettotron"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":380,"score":39.85,"version":"Gemma 3"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":381,"score":36.9,"version":"CodeLlama-7B"},{"company":"IBM","family":"Granite","model":"granite-4-h-300m","place":382,"score":36,"version":"Granite 4"},{"company":"IBM","family":"Granite","model":"granite-4-300m","place":383,"score":35,"version":"Granite 4"},{"company":"loka","family":"TRLM","model":"trlm-135m","place":384,"score":34.95,"version":"TRLM"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":385,"score":32,"version":"Llama 3.2"},{"company":"pleias","family":"Monad","model":"monad","place":386,"score":30,"version":"Monad"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":387,"score":29.9,"version":"Gemma 3"},{"company":"meta","family":"MobileLLM","model":"mobilellm-r1-360m-base","place":388,"score":26.8,"version":"MobileLLM R1"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":389,"score":26.5,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-pt-1b","place":390,"score":26.1,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":391,"score":25,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-instruct","place":392,"score":25,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":393,"score":24.7,"version":"SmolLM2"}],"stars":"1488","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":93.8,"topic":"Multi-domain knowledge","total_models":1613,"url":"https://github.com/hendrycks/test"},{"bench_models":5,"description":"Arabic-language variant of MMLU evaluating knowledge and reasoning.","github":"","has_recent":0,"id":"mmlu-arabic","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLU Arabic","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":1,"score":74.1,"version":"Qwen2.5"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":2,"score":66.8,"version":"Llama 3.1"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":3,"score":65.5,"version":"K2 V2"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":4,"score":65.4,"version":"K2 V2"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":5,"score":65,"version":"Llama 3.0"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":6,"score":47.8,"version":"OLMo 3"}],"stars":"","top_company":"qwen","top_model":"Qwen 2.5 72B","top_model_family":"Qwen","top_model_id":"qwen-2.5-72b","top_model_version":"Qwen2.5","top_open_source":1,"top_percent":74.1,"topic":"Arabic knowledge and reasoning","total_models":1613,"url":""},{"bench_models":3,"description":"Cloze-form MMLU evaluation variant.","github":"https://github.com/hendrycks/test","has_recent":0,"id":"mmlu-cloze","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLU (cloze)","paper":"","placements":[{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":1,"score":31.5,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":2,"score":31.5,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":3,"score":29.3,"version":"SmolLM2"}],"stars":"","top_company":"huggingface","top_model":"SmolLM2 135M Base","top_model_family":"SmolLM","top_model_id":"smollm2-135m-base","top_model_version":"SmolLM2","top_open_source":1,"top_percent":31.5,"topic":"Multi-domain knowledge (cloze)","total_models":1613,"url":"https://github.com/hendrycks/test"},{"bench_models":2,"description":"Full-context MMLU variant evaluating reasoning over long passages.","github":"","has_recent":0,"id":"mmlu-fulltext","leaderboard":"","lower_is_better":false,"metric":"norm_log_acc","name":"Full Text MMLU","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":1,"score":83,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":2,"score":78.6,"version":"Llama TFree HAT"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B Instruct","top_model_family":"Llama","top_model_id":"llama-3.3-70b-instruct","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":83,"topic":"Multi-domain knowledge (long-form)","total_models":1613,"url":""},{"bench_models":298,"description":"Harder successor to MMLU with more challenging questions.","github":"https://github.com/TIGER-AI-Lab/MMLU-Pro","has_recent":0,"id":"mmlu-pro","leaderboard":"https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro","lower_is_better":false,"metric":"percent","name":"MMLU-Pro","paper":"https://arxiv.org/abs/2406.01574","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":90.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":90,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":90,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":89.8,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":89.5,"version":"Claude 4.5"},{"company":"openai","family":"o1","model":"o1","place":6,"score":89.3,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":7,"score":89.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":89.3,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":89.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":10,"score":89,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":11,"score":88.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":88.2,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":13,"score":88,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":88,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":15,"score":87.8,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":16,"score":87.8,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":17,"score":87.7,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":18,"score":87.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":19,"score":87.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":20,"score":87.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":21,"score":87.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":22,"score":87.4,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":23,"score":87.3,"version":"Claude 4.1"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-t1","place":24,"score":87.2,"version":"Hunyuan T1"},{"company":"openai","family":"GPT","model":"gpt-5","place":25,"score":87.1,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":26,"score":87.1,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":27,"score":87.1,"version":"Kimi K2.5"},{"company":"xai","family":"Grok","model":"grok-4","place":28,"score":87,"version":"Grok 4"},{"company":"bytedance","family":"Seed","model":"seed1.5-thinking","place":29,"score":87,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":30,"score":87,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":31,"score":87,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":32,"score":87,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":33,"score":87,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":34,"score":86.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":35,"score":86.7,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":36,"score":86.7,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":37,"score":86.6,"version":"Seed 1.6"},{"company":"xai","family":"Grok","model":"grok-4","place":38,"score":86.6,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":39,"score":86.6,"version":"Claude 4"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":40,"score":86.6,"version":"Intern-S1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":41,"score":86.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":42,"score":86.1,"version":"GPT-4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":43,"score":86.1,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":44,"score":86,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":45,"score":86,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":46,"score":85.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":47,"score":85.9,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":48,"score":85.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":49,"score":85.3,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":50,"score":85.1,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":51,"score":85,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":52,"score":85,"version":"DeepSeek V3.2"},{"company":"openai","family":"o3","model":"o3-high","place":53,"score":85,"version":"o3"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":54,"score":85,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":55,"score":85,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":56,"score":85,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3","place":57,"score":85,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":58,"score":85,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":59,"score":85,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":60,"score":84.9,"version":"DeepSeek R1"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":61,"score":84.9,"version":"Seed 1.8"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":62,"score":84.9,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":63,"score":84.8,"version":"DeepSeek V3.1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":64,"score":84.8,"version":"openPangu-R"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":65,"score":84.69,"version":"Cogito 671B v2.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":66,"score":84.6,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":67,"score":84.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":68,"score":84.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":69,"score":84.3,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":70,"score":84.3,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":71,"score":84.2,"version":"Claude 4"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":72,"score":84.2,"version":"openPangu-R"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":73,"score":84.1,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":74,"score":84,"version":"DeepSeek R1"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":75,"score":83.8,"version":"K-EXAONE"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":76,"score":83.7,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":77,"score":83.7,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":78,"score":83.6,"version":"Seed 2.0"},{"company":"zai","family":"GLM","model":"glm-4.5","place":79,"score":83.5,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":80,"score":83.4,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":81,"score":83.2,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":82,"score":83.2,"version":"GLM 4.6"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":83,"score":83,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":84,"score":82.93,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":85,"score":82.7,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":86,"score":82.4,"version":"Qwen3 Next"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":87,"score":82.4,"version":"Llama Nemotron v1.5"},{"company":"meta","family":"Llama","model":"llama-4-behemoth","place":88,"score":82.2,"version":"Llama 4"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":89,"score":82,"version":"MiniMax M2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":90,"score":81.9,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":91,"score":81.9,"version":"Kimi K2"},{"company":"primeintellect","family":"INTELLECT","model":"intellect-3","place":92,"score":81.9,"version":"INTELLECT"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":93,"score":81.8,"version":"EXAONE 4.0"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":94,"score":81.8,"version":"GPT-4.1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":95,"score":81.8,"version":"EXAONE 4.0"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":96,"score":81.5,"version":"Magistral 1.2"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":97,"score":81.5,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":98,"score":81.4,"version":"GLM 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":99,"score":81.2,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":100,"score":81.2,"version":"DeepSeek V3"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":101,"score":81.1,"version":"MiniMax M1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":102,"score":81.1,"version":"Kimi K2"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":103,"score":81.02,"version":"JoyAI-LLM"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":104,"score":81,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":105,"score":80.9,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":106,"score":80.9,"version":"Llama 4"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":107,"score":80.89,"version":"TeleChat3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":108,"score":80.8,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":109,"score":80.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":110,"score":80.7,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":111,"score":80.7,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":112,"score":80.6,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":113,"score":80.6,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":114,"score":80.6,"version":"MiniMax M1"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":115,"score":80.5,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":116,"score":80.5,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":117,"score":80.5,"version":"Qwen3 VL"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":118,"score":80.4,"version":"Solar Open"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":119,"score":80.4,"version":"GPT-OSS"},{"company":"openai","family":"o1","model":"o1-mini","place":120,"score":80.3,"version":"o1"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":121,"score":80,"version":"Nemotron OpenReasoning"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":122,"score":79.7,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":123,"score":79.4,"version":"Gemini 2.5"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":124,"score":79.2,"version":"Phi-4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":125,"score":79.19,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":126,"score":79.1,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":127,"score":79.1,"version":"Gemini 2.0"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":128,"score":79,"version":"QwQ"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":129,"score":79,"version":"HyperCLOVA X"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":130,"score":78.95,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":131,"score":78.6,"version":"Nemotron-H"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":132,"score":78.6,"version":"GPT-OSS"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":133,"score":78.5,"version":"TeleChat3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":134,"score":78.5,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":135,"score":78.4,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":136,"score":78.4,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":137,"score":78.3,"version":"Nemotron 3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":138,"score":78.29,"version":"LongCat-Flash-Lite"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":139,"score":77.8,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":140,"score":77.6,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":141,"score":77.6,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":142,"score":77.55,"version":"Ling 2.0"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":143,"score":77.5,"version":"Nemotron OpenReasoning"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":144,"score":77.3,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":145,"score":77.3,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":146,"score":77.2,"version":"Mistral 3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":147,"score":77,"version":"Nemotron Cascade"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":148,"score":76.84,"version":"Ling 2.0"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":149,"score":76.8,"version":"Magistral 1.2"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":150,"score":76.8,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":151,"score":76.67,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":152,"score":76.59,"version":"LLaDA 2.1"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":153,"score":76.5,"version":"Apriel 1.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":154,"score":76.48,"version":"Apriel 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":155,"score":75.8,"version":"Gemini 1.5"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":156,"score":75.7,"version":"Nemotron Cascade"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":157,"score":75.6,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":158,"score":75.5,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":159,"score":75.37,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":160,"score":75.31,"version":"LLaDA 2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":161,"score":75.3,"version":"DeepSeek R1"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking","place":162,"score":75.3,"version":"Kanana 2"},{"company":"arcee","family":"Trinity","model":"trinity-large-preview","place":163,"score":75.2,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":164,"score":75.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":165,"score":75,"version":"GPT-OSS"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":166,"score":74.79,"version":"LLaDA 2.0"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":167,"score":74.3,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":168,"score":74.3,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":169,"score":74.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":170,"score":74.23,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":171,"score":74.21,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":172,"score":74.2,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-thinking-2601","place":173,"score":74.2,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":174,"score":74.08,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":175,"score":73.9,"version":"EXAONE 4.0"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":176,"score":73.9,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":177,"score":73.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":178,"score":73.88,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":179,"score":73.6,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":180,"score":73.4,"version":"Llama 3.1"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":181,"score":73.4,"version":"Apriel Nemotron"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":182,"score":73.4,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":183,"score":73.36,"version":"LLaDA 2.0"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":184,"score":73.3,"version":"Nemotron Nano v2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":185,"score":73.26,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":186,"score":73.2,"version":"MiMo V2"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash-base","place":187,"score":73.14,"version":"JoyAI-LLM"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":188,"score":72.7,"version":"Kimi Linear"},{"company":"openbmb","family":"MiniCPM","model":"minicpm4.1-8b","place":189,"score":72.7,"version":"MiniCPM 4.1"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":190,"score":72.1,"version":"Falcon H1R"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":191,"score":71.98,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":192,"score":71.9,"version":"Nemotron OpenReasoning"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":193,"score":71.9,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":194,"score":71.79,"version":"Nemotron Nano V2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":195,"score":71.6,"version":"Qwen3 VL"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":196,"score":70.98,"version":"Falcon H1R"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":197,"score":70.5,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":198,"score":70.41,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":199,"score":70.14,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":200,"score":70,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":201,"score":69.98,"version":"Seed 1.6"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":202,"score":69.6,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":203,"score":69.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4o","place":204,"score":69.1,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":205,"score":69.1,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":206,"score":69.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-max","place":207,"score":69,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":208,"score":69,"version":"GPT-4"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":209,"score":68.9,"version":"Llama 3.3"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":210,"score":68.75,"version":"Ministral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":211,"score":68.18,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":212,"score":67.5,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":213,"score":67.45,"version":"Mistral 3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":214,"score":67.3,"version":"Gemini 1.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":215,"score":67.22,"version":"Kimi Linear"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":216,"score":67.18,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":217,"score":67.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":218,"score":67.1,"version":"GPT-OSS"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":219,"score":67.04,"version":"MiniCPM SALA"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":220,"score":66.63,"version":"Gemma 3"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":221,"score":66.47,"version":"Seed 1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":222,"score":66.4,"version":"Llama 3.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":223,"score":66.38,"version":"Motif 2"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":224,"score":66.02,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":225,"score":65.83,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":226,"score":65.68,"version":"Llama 3.3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":227,"score":65.6,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":228,"score":65.54,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-21b-a3b-pt","place":229,"score":65.5,"version":"ERNIE 4.5"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":230,"score":65.1,"version":"Ling 2.0"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":231,"score":65.05,"version":"Nemotron 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":232,"score":64.84,"version":"LLaDA 2.1"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":233,"score":64.8,"version":"MiMo VL"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":234,"score":64.27,"version":"LLaDA 2.0"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":235,"score":64.09,"version":"Mistral 3.2"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":236,"score":63.91,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4","place":237,"score":63.7,"version":"GPT-4"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":238,"score":63.62,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":239,"score":63.5,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":240,"score":63.42,"version":"LLaDA 2.1"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":241,"score":63.4,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":242,"score":63.4,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-base","place":243,"score":62.9,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":244,"score":62.9,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":245,"score":62.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":246,"score":62.1,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":247,"score":61.83,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-base","place":248,"score":61.76,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":249,"score":61.6,"version":"Llama 3.1"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":250,"score":61.6,"version":"Youtu-LLM"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":251,"score":61.5,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":252,"score":61.49,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":253,"score":61.03,"version":"Qwen3"},{"company":"ai21","family":"Jamba","model":"jamba-reasoning-3b","place":254,"score":61,"version":"Jamba Reasoning"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":255,"score":60.6,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":256,"score":60.21,"version":"Gemma 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":257,"score":59.84,"version":"DeepSeek V3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":258,"score":59.8,"version":"MiMo VL"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":259,"score":59.3,"version":"EXAONE 4.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":260,"score":58.8,"version":"DeepSeek V3.1"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":261,"score":58.73,"version":"Falcon H1"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":262,"score":58.3,"version":"InternVL"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-base","place":263,"score":58.2,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-base","place":264,"score":58.2,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":265,"score":58.1,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":266,"score":58.1,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":267,"score":57.9,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":268,"score":57.8,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":269,"score":57.7,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":270,"score":57.5,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":271,"score":57.3,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":272,"score":57,"version":"Qwen3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":273,"score":57,"version":"K2 V2"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":274,"score":56.9,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":275,"score":56.68,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":276,"score":56.63,"version":"Qwen2.5"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":277,"score":56.4,"version":"EXAONE Deep"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":278,"score":56.14,"version":"Kanana 2"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":279,"score":55.73,"version":"Ouro"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":280,"score":55.47,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":281,"score":55,"version":"Llama 3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":282,"score":54.9,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":283,"score":54.8,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":284,"score":54.68,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b-exp","place":285,"score":53.87,"version":"LFM2"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":286,"score":53.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":287,"score":53.8,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":288,"score":53.72,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":289,"score":53.62,"version":"SmollM3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":290,"score":53.3,"version":"Mi:dm 2.0"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":291,"score":53.1,"version":"Nemotron Nano v2"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":292,"score":53.01,"version":"Llama 3.1"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":293,"score":53,"version":"SmollM3"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":294,"score":52.8,"version":"Llama 3.0"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":295,"score":52.61,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-nothinking-2504","place":296,"score":52.5,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":297,"score":52.4,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":298,"score":52.31,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":299,"score":52.2,"version":"Gemma 3"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":300,"score":52.11,"version":"Solar Pro"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":301,"score":51.75,"version":"Falcon H1"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":302,"score":51.6,"version":"Trinity Large"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":303,"score":51.4,"version":"Qwen3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":304,"score":51.3,"version":"Llama TFree HAT"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":305,"score":51.25,"version":"OLMo 2"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":306,"score":50.85,"version":"EuroLLM"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":307,"score":49.65,"version":"LFM2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":308,"score":49.6,"version":"OLMo 3"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":309,"score":49.4,"version":"Gemma 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":310,"score":49.22,"version":"LLaDA 2.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":311,"score":49.21,"version":"Gemma 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":312,"score":49.2,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":313,"score":48.7,"version":"Qwen2.5 VL"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":314,"score":48.62,"version":"Ouro"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":315,"score":48.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":316,"score":47.6,"version":"Llama 3.1"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":317,"score":47.59,"version":"LFM2"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":318,"score":47.51,"version":"Phi 3"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":319,"score":47.5,"version":"Nemotron OpenReasoning"},{"company":"microsoft","family":"Phi","model":"phi-4-mini","place":320,"score":47,"version":"Phi 4"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":321,"score":46.99,"version":"Phi 3.5 MoE"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":322,"score":46.8,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":323,"score":45.68,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":324,"score":45.5,"version":"Llama 3"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":325,"score":45.33,"version":"EuroLLM"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":326,"score":45.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":327,"score":45.3,"version":"Gemma 3"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":328,"score":45.1,"version":"Marin 32B"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":329,"score":44.94,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":330,"score":44.7,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":331,"score":44.5,"version":"Granite 4.0"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":332,"score":44.35,"version":"LFM2.5"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":333,"score":44.05,"version":"Falcon 3"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":334,"score":43.8,"version":"Moonlight"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":335,"score":43.7,"version":"K2 V2"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":336,"score":43.69,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":337,"score":43.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":338,"score":43.55,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":339,"score":43.48,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":340,"score":43.24,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":341,"score":43.23,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":342,"score":42.91,"version":"Qwen3"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":343,"score":42.4,"version":"Moonlight"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":344,"score":42,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":345,"score":42,"version":"OLMo 2"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":346,"score":41.94,"version":"Apertus"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":347,"score":41.9,"version":"Marin 32B"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":348,"score":41.89,"version":"Falcon H1"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":349,"score":41.65,"version":"Granite 4.0"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":350,"score":40.73,"version":"Falcon 3"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":351,"score":40.7,"version":"EXAONE 3.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":352,"score":39.9,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":353,"score":39.88,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-it","place":354,"score":39.76,"version":"Gemma 3n"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":355,"score":38.3,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":356,"score":37.88,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":357,"score":37.87,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":358,"score":37.8,"version":"Falcon H1"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":359,"score":37.42,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":360,"score":37.27,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":361,"score":37.1,"version":"Llama 3.1"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":362,"score":36.5,"version":"Apertus"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":363,"score":36.5,"version":"DeepSeek R1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":364,"score":36.42,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":365,"score":36.2,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":366,"score":35.98,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":367,"score":35,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":368,"score":34.76,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":369,"score":34.64,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":370,"score":34.61,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":371,"score":33.53,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":372,"score":33.34,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":373,"score":32.76,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":374,"score":32.03,"version":"Granite 4.0"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":375,"score":31.03,"version":"Falcon H1"},{"company":"arcee","family":"Llama Spark","model":"llama-spark-8b","place":376,"score":30.23,"version":"Llama Spark 8B"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":377,"score":29.75,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":378,"score":29.71,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":379,"score":29.71,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":380,"score":29.11,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":381,"score":28.35,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":382,"score":27.64,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":383,"score":27.44,"version":"Llama 3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":384,"score":27,"version":"DeepSeek R1"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":385,"score":25.96,"version":"LFM2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":386,"score":23.9,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":387,"score":23.54,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":388,"score":22.25,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":389,"score":20.8,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":390,"score":20.8,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":391,"score":18.73,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":392,"score":18.49,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-e-1b-base","place":393,"score":17.82,"version":"Falcon E"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":394,"score":16.95,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":395,"score":16.2,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":396,"score":15.6,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":397,"score":14.7,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":398,"score":14.46,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":399,"score":14.04,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":400,"score":13.98,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":401,"score":11.8,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":402,"score":11.31,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":403,"score":3.91,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":404,"score":3.91,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":405,"score":1.88,"version":"SmolLM2"}],"stars":"286","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":90.1,"topic":"Multi-domain knowledge","total_models":1613,"url":"https://github.com/TIGER-AI-Lab/MMLU-Pro"},{"bench_models":9,"description":"MMLU-Pro common format (MCF) few-shot evaluation.","github":"","has_recent":0,"id":"mmlu-pro-mcf","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLU Pro MCF","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-4b-base","place":1,"score":41.07,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":2,"score":24.2,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":3,"score":19.4,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":4,"score":19.3,"version":"SmolLM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":5,"score":13.7,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-1b","place":6,"score":12.7,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":7,"score":11.7,"version":"Llama 1"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":8,"score":11.7,"version":"SmolLM1"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":9,"score":10.8,"version":"SmolLM1"}],"stars":"","top_company":"qwen","top_model":"Qwen3-4B-Base","top_model_family":"Qwen","top_model_id":"qwen3-4b-base","top_model_version":"Qwen3","top_open_source":1,"top_percent":41.07,"topic":"Multi-domain knowledge (few-shot)","total_models":1613,"url":""},{"bench_models":30,"description":"Cross-lingual and robust variant of MMLU-Pro.","github":"","has_recent":0,"id":"mmlu-prox","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLU-ProX","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":87.7,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":85.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":84.7,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":83.7,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":5,"score":82.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":6,"score":82.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":7,"score":82.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":8,"score":81,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":9,"score":81,"version":"Qwen3.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":10,"score":80.6,"version":"openPangu-R"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":11,"score":80.2,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":12,"score":79.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":13,"score":78.7,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":14,"score":78.5,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":15,"score":78.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":16,"score":77.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":17,"score":77.6,"version":"Qwen3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":18,"score":76.9,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":19,"score":76.7,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":20,"score":76.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":21,"score":76.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":22,"score":74.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":23,"score":74.5,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":24,"score":72,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":25,"score":70.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":26,"score":69.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":27,"score":69.1,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":28,"score":65.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":29,"score":61.6,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":30,"score":59.5,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":31,"score":59.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":32,"score":58,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":87.7,"topic":"Multi-domain knowledge","total_models":1613,"url":""},{"bench_models":74,"description":"Updated MMLU-style evaluation with revised questions and scoring.","github":"","has_recent":0,"id":"mmlu-redux","leaderboard":"https://huggingface.co/spaces/allenai/ZeroEval","lower_is_better":false,"metric":"percent","name":"MMLU-Redux","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":95.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":2,"score":95.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":95.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":95.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":5,"score":95,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":95,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":7,"score":94.9,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":8,"score":94.5,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":9,"score":94.5,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":10,"score":94.4,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":11,"score":94.2,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":12,"score":94,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":13,"score":93.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":14,"score":93.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":15,"score":93.7,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":16,"score":93.7,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":17,"score":93.7,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":18,"score":93.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":19,"score":93.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":20,"score":93.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":21,"score":93.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":22,"score":92.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":23,"score":92.7,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":24,"score":92.7,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":25,"score":92.7,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":26,"score":92.5,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":27,"score":92.4,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":28,"score":92.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":29,"score":91.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":30,"score":91.3,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":31,"score":91,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":32,"score":90.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":33,"score":90.9,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":34,"score":90.9,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":35,"score":90.6,"version":"Gemini 2.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":36,"score":90.6,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":37,"score":90.5,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":38,"score":90.4,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":39,"score":90.2,"version":"Kimi K2"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":40,"score":90.02,"version":"Motif 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":41,"score":90,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":42,"score":89.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":43,"score":89.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":44,"score":89.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":45,"score":89.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":46,"score":89.2,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":47,"score":88.89,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":48,"score":88.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":49,"score":88.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":50,"score":88.4,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":51,"score":86.9,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":52,"score":86.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":53,"score":86.8,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":54,"score":86.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":55,"score":85.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":56,"score":84.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":57,"score":84.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":58,"score":84.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":59,"score":83.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":60,"score":83.41,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":61,"score":82,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":62,"score":81.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":63,"score":81.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":64,"score":81.17,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":65,"score":79.88,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":66,"score":79.7,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":67,"score":79.5,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":68,"score":78.68,"version":"Motif 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":69,"score":78.1,"version":"DeepSeek R1"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":70,"score":75.8,"version":"Youtu-LLM"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":71,"score":75.6,"version":"SmollM3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":72,"score":74.8,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":73,"score":74.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":74,"score":73.9,"version":"Qwen3"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":75,"score":73.8,"version":"Moonlight"},{"company":"radicalnumerics","family":"RND1","model":"rnd1-base-0910","place":76,"score":72.6,"version":"RND1"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":77,"score":71.5,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":78,"score":68.9,"version":"EXAONE Deep"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-instruct","place":79,"score":66.9,"version":"Moonlight"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":80,"score":55.6,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":81,"score":53,"version":"DeepSeek R1"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":95.9,"topic":"Multi-domain knowledge","total_models":1613,"url":""},{"bench_models":30,"description":"STEM subset of MMLU.","github":"","has_recent":0,"id":"mmlu-stem","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMLU-STEM","paper":"","placements":[{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":1,"score":83.57,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":2,"score":82.59,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":3,"score":82.37,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":4,"score":81.64,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":5,"score":77.61,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":6,"score":75.2,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":7,"score":74.88,"version":"Llama 3.3"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":8,"score":73.55,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":9,"score":70.57,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":10,"score":69.93,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":11,"score":69.36,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":12,"score":67.46,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":13,"score":67.43,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":14,"score":67.3,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":15,"score":66.89,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":16,"score":66.54,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":17,"score":64.13,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":18,"score":59.78,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":19,"score":59.31,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":20,"score":56.11,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":21,"score":54.55,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":22,"score":54.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":23,"score":54.04,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":24,"score":52.17,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":25,"score":51.92,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":26,"score":39.83,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":27,"score":39.64,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":28,"score":39.3,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":29,"score":39.16,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":30,"score":35.39,"version":"Gemma 3"}],"stars":"1488","top_company":"tii","top_model":"Falcon-H1-34B-Instruct","top_model_family":"Falcon","top_model_id":"falcon-h1-34b-instruct","top_model_version":"Falcon H1","top_open_source":1,"top_percent":83.57,"topic":"STEM knowledge","total_models":1613,"url":"https://github.com/hendrycks/test"},{"bench_models":4,"description":"Multilingual Multimodal Benchmark (MMMB) average score.","github":"","has_recent":0,"id":"mmmb","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMMB","paper":"","placements":[{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":1,"score":76.96,"version":"LFM2.5-VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":2,"score":72.13,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":3,"score":68.93,"version":"InternVL 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":4,"score":64.84,"version":"FastVLM"}],"stars":"","top_company":"liquidai","top_model":"LFM2.5-VL-1.6B","top_model_family":"LFM","top_model_id":"lfm2.5-vl-1.6b","top_model_version":"LFM2.5-VL","top_open_source":1,"top_percent":76.96,"topic":"Multilingual MMBench","total_models":1613,"url":""},{"bench_models":59,"description":"Massively multilingual MMLU-style evaluation across many languages.","github":"","has_recent":0,"id":"mmmlu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMMLU","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":92.6,"version":"Gemini 3.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":91.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":91.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":91.8,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":5,"score":91.1,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":6,"score":91.1,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":7,"score":91,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":91,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":90.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":90.8,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":11,"score":90.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":12,"score":90.3,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":13,"score":90.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":89.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":15,"score":89.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":16,"score":89.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":17,"score":89.6,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":18,"score":89.5,"version":"Claude 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":19,"score":89.5,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":20,"score":89.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":21,"score":89.5,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":22,"score":89.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":23,"score":89.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":24,"score":89.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":25,"score":89.3,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":26,"score":89.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":27,"score":89.1,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":28,"score":88.5,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":29,"score":88.1,"version":"Seed 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":30,"score":88,"version":"DeepSeek V3.2"},{"company":"openai","family":"o1","model":"o1","place":31,"score":87.7,"version":"o1"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":32,"score":87.7,"version":"Seed 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":33,"score":87.3,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":34,"score":86.8,"version":"Grok 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":35,"score":86.7,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":36,"score":86.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":37,"score":86.5,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":38,"score":86.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":39,"score":86.2,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":40,"score":86,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":41,"score":85.9,"version":"Qwen3.5"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":42,"score":85.7,"version":"K-EXAONE"},{"company":"mistral","family":"Mistral","model":"mistral-large-3-base","place":43,"score":85.5,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":44,"score":85.2,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":45,"score":85.1,"version":"GPT-4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":46,"score":84.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":47,"score":83.8,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":48,"score":83.4,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":49,"score":83.2,"version":"EXAONE 4.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":50,"score":81.6,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":51,"score":78.2,"version":"GPT-OSS"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":52,"score":69.69,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":53,"score":60.67,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":54,"score":57.37,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":55,"score":56.59,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":56,"score":56.13,"version":"Granite 4.0"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":57,"score":55.39,"version":"LFM2"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":58,"score":55.26,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":59,"score":55.14,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":60,"score":50.14,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":61,"score":50.02,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":62,"score":47.92,"version":"Llama 3.2"},{"company":"liquidai","family":"LFM","model":"lfm2-1.2b","place":63,"score":46.73,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":64,"score":46.51,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-700m","place":65,"score":43.28,"version":"LFM2"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b","place":66,"score":38.15,"version":"Llama 3.2"},{"company":"liquidai","family":"LFM","model":"lfm2-350m","place":67,"score":37.99,"version":"LFM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":68,"score":34.43,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":69,"score":30.84,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":70,"score":0.783,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":71,"score":0.715,"version":"Llama TFree HAT"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":92.6,"topic":"Multi-domain knowledge (multilingual)","total_models":1613,"url":"https://huggingface.co/LiquidAI/LFM2-350M"},{"bench_models":5,"description":"Spanish MMMLU benchmark","github":"","has_recent":0,"id":"mmmlu-es","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMMLU (ES)","paper":"","placements":[{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":1,"score":64.7,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":2,"score":64.5,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":3,"score":62.4,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":4,"score":51.4,"version":"EXAONE Deep"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":5,"score":48.8,"version":"Qwen3"}],"stars":"","top_company":"huggingface","top_model":"SmolLM 3 3B","top_model_family":"SmolLM","top_model_id":"smollm-3-3b","top_model_version":"SmolLM3","top_open_source":1,"top_percent":64.7,"topic":"Multilingual knowledge","total_models":1613,"url":""},{"bench_models":157,"description":"Multi-discipline multimodal understanding benchmark.","github":"","has_recent":0,"id":"mmmu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMMU","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":87.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":87,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":86.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":4,"score":85.4,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":85.2,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":6,"score":85,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":84.3,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":84.22,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":9,"score":84.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":10,"score":83.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":83.89,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":12,"score":83.4,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":83.3,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3","place":14,"score":82.9,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":15,"score":82.3,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":82,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":17,"score":81.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":81.7,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":19,"score":81.4,"version":"Qwen3.5"},{"company":"xai","family":"Grok","model":"grok-4","place":20,"score":80.9,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":21,"score":80.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":22,"score":80.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":23,"score":80.6,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":24,"score":80.11,"version":"dots.vlm1"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":25,"score":80.11,"version":"Step3-VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":26,"score":79.89,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":27,"score":79.8,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":28,"score":79.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":29,"score":79.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":30,"score":79.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":31,"score":79.11,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":32,"score":79,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":33,"score":79,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":34,"score":78.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":35,"score":78.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":36,"score":78.7,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":37,"score":78.11,"version":"Step3-VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":38,"score":78.11,"version":"Step3-VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":39,"score":77.9,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":40,"score":77.8,"version":"Claude 4.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":41,"score":77.2,"version":"Ovis 2.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":42,"score":77.1,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":43,"score":76.9,"version":"Claude 4"},{"company":"meta","family":"Llama","model":"llama-4-behemoth","place":44,"score":76.1,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":45,"score":76,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":46,"score":76,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":47,"score":76,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":48,"score":75.4,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":49,"score":75.2,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":50,"score":75.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":51,"score":75.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":52,"score":74.4,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":53,"score":74.4,"version":"GPT-4.5"},{"company":"stepfun","family":"Step","model":"step-3","place":54,"score":74.2,"version":"Step 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":55,"score":74.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":56,"score":73.53,"version":"Qwen3 VL"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":57,"score":73.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":58,"score":73.4,"version":"Llama 4"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":59,"score":73.4,"version":"InternVL 3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":60,"score":73.2,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":61,"score":73.2,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":62,"score":72.8,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":63,"score":72.7,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":64,"score":72.7,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-4o","place":65,"score":72.2,"version":"GPT-4o"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":66,"score":72.2,"version":"ERNIE 4.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":67,"score":71.7,"version":"Gemini 2.0"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":68,"score":71.69,"version":"InternVL 3.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":69,"score":71.4,"version":"Keye-VL 1.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":70,"score":71.17,"version":"GLM 4.6V"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":71,"score":71.14,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":72,"score":71.1,"version":"GLM 4.6V"},{"company":"qwen","family":"QvQ","model":"qvq-72b","place":73,"score":70.3,"version":"QvQ 72B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":74,"score":70.2,"version":"Qwen2.5 VL"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":75,"score":70,"version":"ERNIE 4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":76,"score":69.8,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":77,"score":69.6,"version":"Qwen3 VL"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":78,"score":69.4,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":79,"score":69.4,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":80,"score":69.3,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":81,"score":69.2,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":82,"score":69.1,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o","place":83,"score":68.6,"version":"GPT-4o"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":84,"score":68.6,"version":"MolmoE"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":85,"score":68.3,"version":"Claude 3.5"},{"company":"tencent","family":"R","model":"r-4b-base","place":86,"score":68.1,"version":"R"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":87,"score":68,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":88,"score":68,"version":"Claude 3.5"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":89,"score":68,"version":"GLM 4.1V"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":90,"score":67.9,"version":"GPT-5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":91,"score":67.7,"version":"MiniCPM V 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":92,"score":67.4,"version":"Qwen3 VL"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":93,"score":66.8,"version":"Keye-VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl","place":94,"score":66.7,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":95,"score":66.7,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":96,"score":66.6,"version":"InternVL 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":97,"score":66.1,"version":"Mistral 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":98,"score":65.9,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":99,"score":64.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":100,"score":64.9,"version":"Gemma 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":101,"score":64.6,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":102,"score":64.5,"version":"Qwen2 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":103,"score":64,"version":"Kimi VL"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":104,"score":64,"version":"Mistral 3.1"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":105,"score":64,"version":"Pixtral"},{"company":"openai","family":"GPT","model":"gpt-4v","place":106,"score":63.1,"version":"GPT-4V"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":107,"score":62.7,"version":"InternVL"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":108,"score":62.5,"version":"Mistral 3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":109,"score":62.3,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":110,"score":62.2,"version":"Gemini 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl3-8b","place":111,"score":62.2,"version":"InternVL 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":112,"score":62,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":113,"score":60.6,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":114,"score":60.3,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":115,"score":59.6,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":116,"score":59.4,"version":"Claude 3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":117,"score":59,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":118,"score":58.6,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":119,"score":58.2,"version":"InternVL 2"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":120,"score":57.8,"version":"Manzano"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":121,"score":57.6,"version":"GPT-5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":122,"score":56.8,"version":"LLaVA OneVision"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":123,"score":56.1,"version":"Gemini 1.5"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":124,"score":55.8,"version":"Eagle 2.5"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":125,"score":55.3,"version":"Bagel"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":126,"score":54.4,"version":"LLaVA OneVision"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":127,"score":54.1,"version":"Molmo"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":128,"score":54.1,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":129,"score":53.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":130,"score":53.1,"version":"Qwen2.5 VL"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":131,"score":53,"version":"Molmo2"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":132,"score":52.5,"version":"Pixtral"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":133,"score":52.3,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":134,"score":51.8,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":135,"score":51.78,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":136,"score":51.67,"version":"Qwen2.5 VL"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":137,"score":51.4,"version":"Manzano"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":138,"score":51.2,"version":"InternVL 2"},{"company":"bytedance","family":"LLaVA","model":"llava-next-34b","place":139,"score":51.1,"version":"LLaVA-NeXT"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":140,"score":50.9,"version":"Molmo2"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":141,"score":50.7,"version":"Llama 3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-0827","place":142,"score":50.7,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":143,"score":50.4,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":144,"score":50.2,"version":"Claude 3"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":145,"score":49.7,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":146,"score":49.7,"version":"Cambrian"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":147,"score":48.8,"version":"LLaVA OneVision"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":148,"score":48.8,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":149,"score":48.7,"version":"InternVL 3"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":150,"score":47.4,"version":"MM1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":151,"score":47.2,"version":"Qwen2.5 Omni"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":152,"score":46.6,"version":"BLIP-3O"},{"company":"unknown","family":"PLM","model":"plm-8b","place":153,"score":46.1,"version":"PLM"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":154,"score":45.8,"version":"Molmo2"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":155,"score":45.4,"version":"FastVLM"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":156,"score":45.33,"version":"LFM2-VL"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":157,"score":45.3,"version":"Molmo"},{"company":"apple","family":"MM1","model":"mm1-30b","place":158,"score":44.7,"version":"MM1"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":159,"score":43.6,"version":"InternVL 2.5"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":160,"score":43,"version":"Phi 3.5"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":161,"score":42.7,"version":"Cambrian 1"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":162,"score":41.89,"version":"InternVL 3.5"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":163,"score":41.6,"version":"SmolVLM2"},{"company":"baai","family":"Bunny","model":"bunny-4b","place":164,"score":41.4,"version":"Bunny"},{"company":"unknown","family":"PLM","model":"plm-3b","place":165,"score":41.2,"version":"PLM"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":166,"score":41.1,"version":"xGen-MM"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":167,"score":41.1,"version":"BLIP-3"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":168,"score":41,"version":"Janus Pro"},{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":169,"score":40.56,"version":"LFM2.5-VL"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":170,"score":40.4,"version":"Phi-3"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":171,"score":39.67,"version":"LFM2-VL"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":172,"score":39.3,"version":"Molmo"},{"company":"harmonai","family":"Harmon","model":"harmon-1.5b","place":173,"score":38.9,"version":"Harmon-1.5B"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":174,"score":38.78,"version":"FastVLM"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":175,"score":38.2,"version":"MiniCPM V 2.0"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":176,"score":37.8,"version":"FastVLM"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":177,"score":37.1,"version":"MM1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":178,"score":37,"version":"LLaVA 1.5"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":179,"score":36.3,"version":"Janus Pro"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":180,"score":35.7,"version":"LLaVA 1.5"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":181,"score":34.9,"version":"MolmoE"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":182,"score":34.9,"version":"PaliGemma"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":183,"score":33.9,"version":"FastVLM"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":184,"score":33.3,"version":"VILA 1.5"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":185,"score":32.6,"version":"Gemini Nano"},{"company":"baai","family":"Emu","model":"emu3-8b","place":186,"score":31.6,"version":"Emu3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":87.2,"topic":"Multimodal understanding","total_models":1613,"url":"https://mmmu-benchmark.github.io/"},{"bench_models":68,"description":"Professional/advanced subset of MMMU for multimodal reasoning.","github":"","has_recent":0,"id":"mmmu-pro","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMMU PRO","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-deep-think","place":1,"score":81.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":81.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":81,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":81,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":5,"score":80.5,"version":"Gemini 3.1"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":6,"score":79.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":7,"score":79.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":79.5,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":9,"score":79.5,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":10,"score":79,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":11,"score":78.5,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":12,"score":78.5,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":13,"score":78.4,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":14,"score":76.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":15,"score":76.5,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":16,"score":76.4,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":17,"score":76,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":18,"score":76,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":19,"score":75.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":20,"score":75,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":21,"score":74.5,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":22,"score":74,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":23,"score":73.9,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":24,"score":73.9,"version":"Claude 4.6"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":25,"score":73.2,"version":"Seed 1.8"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":26,"score":72.8,"version":"Intern-S1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":27,"score":70.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":28,"score":70.6,"version":"Claude 4.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":29,"score":70.11,"version":"dots.vlm1"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":30,"score":70.1,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":31,"score":69.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":32,"score":69.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":33,"score":69.3,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":34,"score":68.9,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":35,"score":68.8,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":36,"score":68.4,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-instruct","place":37,"score":68.1,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":38,"score":68,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":39,"score":68,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":40,"score":68,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":41,"score":68,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":42,"score":67.6,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":43,"score":67.3,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":44,"score":66.7,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":45,"score":66,"version":"GLM 4.6V"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":46,"score":65.6,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":47,"score":65.3,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":48,"score":65.2,"version":"GLM 4.5V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":49,"score":64.8,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":50,"score":63.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":51,"score":63.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":52,"score":63.4,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":53,"score":63,"version":"Grok 4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":54,"score":63,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":55,"score":60.6,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":56,"score":60.4,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":57,"score":59.9,"version":"GPT-4o"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":58,"score":59.9,"version":"Seed 1.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":59,"score":59.6,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":60,"score":59.6,"version":"Llama 4"},{"company":"stepfun","family":"Step","model":"step-3","place":61,"score":58.6,"version":"Step 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":62,"score":57.1,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":63,"score":56.5,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":64,"score":56.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":65,"score":56.1,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":66,"score":55.9,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":67,"score":55.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":68,"score":53.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":69,"score":53.2,"version":"Qwen3 VL"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":70,"score":52.2,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":71,"score":52.2,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":72,"score":51.91,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":73,"score":51.1,"version":"Qwen2.5 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":74,"score":46.3,"version":"Kimi VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":75,"score":46.2,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":76,"score":45.6,"version":"InternVL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":77,"score":45.2,"version":"MiMo VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":78,"score":37.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":79,"score":37.4,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":80,"score":36.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":81,"score":36.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":82,"score":34.7,"version":"Qwen2.5 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Deep Think","top_model_family":"Gemini","top_model_id":"gemini-3-deep-think","top_model_version":"Gemini 3","top_open_source":0,"top_percent":81.5,"topic":"Multimodal understanding (hard)","total_models":1613,"url":"https://mmmu-benchmark.github.io/"},{"bench_models":7,"description":"MMMU-Pro vision-only setting.","github":"","has_recent":0,"id":"mmmu-pro-vision","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMMU-Pro (vision)","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":1,"score":45.8,"version":"Claude 3.7"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":2,"score":40.3,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":3,"score":39.4,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":4,"score":37.8,"version":"InternVL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":5,"score":36.1,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":6,"score":29.4,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":7,"score":24.9,"version":"Gemma 3"}],"stars":"","top_company":"anthropic","top_model":"Claude 3.7 Sonnet","top_model_family":"Claude","top_model_id":"claude-3.7-sonnet","top_model_version":"Claude 3.7","top_open_source":0,"top_percent":45.8,"topic":"Multimodal understanding (vision)","total_models":1613,"url":"https://mmmu-benchmark.github.io/"},{"bench_models":6,"description":"MMMU-Pro benchmark evaluated with tool access.","github":"","has_recent":0,"id":"mmmu-pro-with-tools","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMMU Pro (with tools)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":80.4,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":80.4,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":3,"score":77.3,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":4,"score":77.3,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":5,"score":75.6,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":73.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":73.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":8,"score":68.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":68.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":10,"score":68.9,"version":"Claude 4.5"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":80.4,"topic":"Multimodal understanding (with tools)","total_models":1613,"url":"https://mmmu-benchmark.github.io/"},{"bench_models":5,"description":"MMSIBench circular subset for spatial reasoning.","github":"","has_recent":0,"id":"mmsibench-circular","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMSIBench (circular)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":25.4,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":22.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":17.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":17.2,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":5,"score":11.7,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":25.4,"topic":"Spatial understanding","total_models":1613,"url":""},{"bench_models":63,"description":"Broad evaluation of multimodal LLMs across diverse tasks.","github":"https://github.com/MMStar-Benchmark/MMStar","has_recent":0,"id":"mmstar","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMStar","paper":"https://arxiv.org/abs/2403.20330","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":83.8,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":83.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":82.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":4,"score":81.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":5,"score":81,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":80.5,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":79.18,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":78.73,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":9,"score":78.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":10,"score":78.7,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":11,"score":78.33,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":12,"score":77.91,"version":"Seed 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":13,"score":77.9,"version":"InternVL 3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":77.8,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":15,"score":77.8,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":16,"score":77.8,"version":"Seed 1.8"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":17,"score":77.7,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":18,"score":77.64,"version":"Step3-VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":19,"score":77.5,"version":"Gemini 2.5"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":20,"score":77.48,"version":"Step3-VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":21,"score":77.48,"version":"Step3-VL"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":22,"score":77.3,"version":"Ovis 2.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":23,"score":77.1,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":24,"score":76.8,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":25,"score":76.67,"version":"dots.vlm1"},{"company":"openai","family":"GPT","model":"gpt-5","place":26,"score":76.4,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":27,"score":75.9,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":28,"score":75.5,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":29,"score":75.3,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":30,"score":75.3,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":31,"score":74.7,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":32,"score":74.26,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":33,"score":74.1,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":34,"score":74.1,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":35,"score":73.8,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":36,"score":73.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":37,"score":73.58,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":38,"score":73.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":39,"score":73.2,"version":"Claude 4.5"},{"company":"tencent","family":"R","model":"r-4b-base","place":40,"score":73.1,"version":"R"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":41,"score":72.93,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":42,"score":72.9,"version":"GLM 4.1V"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":43,"score":72.8,"version":"Keye-VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":44,"score":72.1,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":45,"score":72.1,"version":"Claude 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":46,"score":71.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":47,"score":71.13,"version":"Qwen2.5 VL"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":48,"score":71.1,"version":"ERNIE 4.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":49,"score":70.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":50,"score":70.8,"version":"Qwen2.5 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":51,"score":70.4,"version":"Kimi VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":52,"score":69.83,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":53,"score":69.8,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step-3","place":54,"score":69,"version":"Step 3"},{"company":"opengvlab","family":"Intern","model":"internvl3-8b","place":55,"score":68.7,"version":"InternVL 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":56,"score":67.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":57,"score":67.4,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":58,"score":64.1,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":59,"score":64.1,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":60,"score":61.1,"version":"InternVL 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":61,"score":60,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":62,"score":58.3,"version":"Qwen3 VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":63,"score":57.73,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":64,"score":57.67,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":65,"score":56.13,"version":"Qwen2.5 VL"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":66,"score":53.13,"version":"FastVLM"},{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":67,"score":50.67,"version":"LFM2.5-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":68,"score":50.27,"version":"InternVL 3.5"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":69,"score":49.87,"version":"LFM2-VL"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":70,"score":46,"version":"SmolVLM2"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":71,"score":41.3,"version":"GPT-5"},{"company":"IBM","family":"Granite","model":"granite-docling-258m","place":72,"score":30,"version":"Granite Docling"},{"company":"IBM","family":"Docling","model":"smoldocling-256m-preview","place":73,"score":17,"version":"SMoLDocling"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":83.8,"topic":"Multimodal reasoning","total_models":1613,"url":"https://mmstar-benchmark.github.io/"},{"bench_models":6,"description":"Comprehensive evaluation suite for assessing multimodal LLM capabilities.","github":"https://github.com/yuweihao/MM-Vet","has_recent":0,"id":"mmvet","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMVet","paper":"https://arxiv.org/abs/2308.02490","placements":[{"company":"tencent","family":"R","model":"r-4b-base","place":1,"score":85.9,"version":"R"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":83.3,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":3,"score":78.8,"version":"GLM 4.6V"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":4,"score":78,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":5,"score":75.4,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":74.6,"version":"GPT-5"}],"stars":"","top_company":"tencent","top_model":"R-4B-Base","top_model_family":"R","top_model_id":"r-4b-base","top_model_version":"R","top_open_source":1,"top_percent":85.9,"topic":"Multimodal evaluation","total_models":1613,"url":"https://mm-vet.github.io/"},{"bench_models":9,"description":"Benchmark for multimodal video understanding and perception.","github":"https://github.com/Metaverse-AI-Lab-THU/MMVP-Dataset","has_recent":0,"id":"mmvp","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMVP","paper":"https://arxiv.org/abs/2308.16154","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":91.6,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":90,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":84.3,"version":"Claude 4.5"},{"company":"tencent","family":"R","model":"r-4b-rl","place":4,"score":80.7,"version":"R"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":74.7,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":73.33,"version":"Seed 1.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":7,"score":72,"version":"dots.vlm1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":70.7,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":9,"score":69.3,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":67.33,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":11,"score":66,"version":"Qwen2.5 VL"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":91.6,"topic":"Multimodal video perception","total_models":1613,"url":"https://mmvp-benchmark.github.io/"},{"bench_models":22,"description":"Multimodal video understanding benchmark (MMVU).","github":"","has_recent":0,"id":"mmvu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMVU","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":1,"score":80.8,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":80.8,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":3,"score":80.4,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":80.4,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":77.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":6,"score":77.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":77.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":76.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":76.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":10,"score":75.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":11,"score":74.7,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":12,"score":73.3,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":13,"score":73.1,"version":"Seed 1.8"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":14,"score":72.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":15,"score":71.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":16,"score":71.1,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":17,"score":70.6,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":18,"score":70.1,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":19,"score":69.8,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":20,"score":68.7,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":21,"score":59.4,"version":"GLM 4.1V"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":22,"score":57.7,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":23,"score":57.5,"version":"Kimi VL"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 Thinking XHigh","top_model_family":"GPT","top_model_id":"gpt-5.2-thinking-xhigh","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":80.8,"topic":"Video understanding","total_models":1613,"url":""},{"bench_models":5,"description":"Instruction-following benchmark for bio-molecular understanding and generation.","github":"","has_recent":0,"id":"mol-instructions","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Mol-Instructions","paper":"","placements":[{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":1,"score":48.8,"version":"Intern-S1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":34.6,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":20,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":12.3,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":8.9,"version":"Qwen3 VL"}],"stars":"","top_company":"internlm","top_model":"Intern-S1-Pro","top_model_family":"Intern","top_model_id":"intern-s1-pro","top_model_version":"Intern-S1","top_open_source":1,"top_percent":48.8,"topic":"Bio-molecular instruction following","total_models":1613,"url":""},{"bench_models":11,"description":"Video motion and temporal reasoning benchmark.","github":"","has_recent":0,"id":"motionbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MotionBench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":70.6,"version":"Seed 1.8"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":2,"score":70.4,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":70.3,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":68.8,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":66.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":6,"score":64.8,"version":"GPT-5.2"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":7,"score":62.4,"version":"GLM 4.5V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":8,"score":60.3,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":9,"score":59,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":10,"score":54.3,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":11,"score":47.8,"version":"Gemma 3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":70.6,"topic":"Video motion understanding","total_models":1613,"url":""},{"bench_models":32,"description":"OpenAI Multi-Round Chain Reasoning benchmark with 128k context window.","github":"","has_recent":0,"id":"mrcr-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OpenAI-MRCR (128k)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":89.7,"version":"Gemini 3"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1.5-30b-a3b","place":2,"score":82.99,"version":"QwenLong"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":3,"score":81.9,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":79.92,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":5,"score":78.84,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":77.29,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":77,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":76.8,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":9,"score":76.1,"version":"MiniMax M1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":10,"score":73.4,"version":"MiniMax M1"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking-preview","place":11,"score":71.24,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":12,"score":67.2,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":13,"score":66.17,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":14,"score":64.88,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":15,"score":61.6,"version":"GPT-5.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":16,"score":58.6,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":17,"score":58,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":18,"score":57.7,"version":"DeepSeek V3.2"},{"company":"openai","family":"o3","model":"o3","place":19,"score":56.5,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":20,"score":55.5,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":21,"score":55.4,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":22,"score":54.6,"version":"Grok 4.1"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":23,"score":54.3,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":24,"score":54.3,"version":"Gemini 2.5"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":25,"score":52.3,"version":"K-EXAONE"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":26,"score":51.5,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":27,"score":51.27,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":28,"score":48.9,"version":"Claude 4"},{"company":"qwen","family":"QwenLong","model":"qwenlong-l1","place":29,"score":47.86,"version":"QwenLong"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":30,"score":47.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":31,"score":47.1,"version":"Claude 4.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":32,"score":45.7,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":33,"score":44.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":34,"score":43.88,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":35,"score":39.68,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":36,"score":35.8,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":37,"score":29.9,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":38,"score":27.7,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":39,"score":20.1,"version":"EXAONE 4.0"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":89.7,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Multi-Round Coreference Resolution benchmark at 128k context with 2 needles.","github":"","has_recent":0,"id":"mrcr-128k-2n","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MRCR 128K-2N","paper":"","placements":[{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":1,"score":50.3,"version":"Ministral 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":2,"score":28.62,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":3,"score":26.5,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":4,"score":14.61,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":5,"score":9.17,"version":"Falcon H1R"}],"stars":"","top_company":"mistral","top_model":"Ministral-3-R 8B","top_model_family":"Ministral","top_model_id":"ministral-3-r-8b","top_model_version":"Ministral 3","top_open_source":1,"top_percent":50.3,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Multi-Round Coreference Resolution benchmark at 128k context with 4 needles.","github":"","has_recent":0,"id":"mrcr-128k-4n","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MRCR 128K-4N","paper":"","placements":[{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":1,"score":22.66,"version":"Ministral 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":2,"score":19.62,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":3,"score":14.75,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":4,"score":12.2,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":5,"score":8.22,"version":"Falcon H1R"}],"stars":"","top_company":"mistral","top_model":"Ministral-3-R 8B","top_model_family":"Ministral","top_model_id":"ministral-3-r-8b","top_model_version":"Ministral 3","top_open_source":1,"top_percent":22.66,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":10,"description":"Multi-Round Coreference Resolution benchmark at 128k context with 8 needles.","github":"","has_recent":0,"id":"mrcr-128k-8n","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MRCR 128K-8N","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":84.9,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":2,"score":84.9,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":3,"score":84,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":83.8,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":77,"version":"Gemini 3"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":6,"score":14.47,"version":"Ministral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":7,"score":12.15,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":8,"score":10.12,"version":"MiniCPM SALA"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":9,"score":7.55,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":10,"score":7.54,"version":"Falcon H1R"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":84.9,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"OpenAI Multi-Round Chain Reasoning benchmark with 1M context window.","github":"","has_recent":0,"id":"mrcr-1m","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OpenAI-MRCR (1M)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":58.8,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":2,"score":58.6,"version":"MiniMax M1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":3,"score":56.2,"version":"MiniMax M1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":26.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":5,"score":26.3,"version":"Gemini 3.1"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":6,"score":22.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":7,"score":21,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":16.4,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":9,"score":9.6,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":10,"score":6.1,"version":"Grok 4.1"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":58.8,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Multi-Round Coreference Resolution benchmark at 64k context with 2 needles.","github":"","has_recent":0,"id":"mrcr-64k-2n","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MRCR 64K-2N","paper":"","placements":[{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":1,"score":44.02,"version":"Ministral 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":2,"score":29.77,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":3,"score":29.2,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":4,"score":20.91,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":5,"score":13.18,"version":"Falcon H1R"}],"stars":"","top_company":"mistral","top_model":"Ministral-3-R 8B","top_model_family":"Ministral","top_model_id":"ministral-3-r-8b","top_model_version":"Ministral 3","top_open_source":1,"top_percent":44.02,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Multi-Round Coreference Resolution benchmark at 64k context with 4 needles.","github":"","has_recent":0,"id":"mrcr-64k-4n","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MRCR 64K-4N","paper":"","placements":[{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":1,"score":35.8,"version":"Ministral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":2,"score":21.56,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":3,"score":20.57,"version":"MiniCPM SALA"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":4,"score":13.69,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":5,"score":9.06,"version":"Falcon H1R"}],"stars":"","top_company":"mistral","top_model":"Ministral-3-R 8B","top_model_family":"Ministral","top_model_id":"ministral-3-r-8b","top_model_version":"Ministral 3","top_open_source":1,"top_percent":35.8,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Multi-Round Coreference Resolution benchmark at 64k context with 8 needles.","github":"","has_recent":0,"id":"mrcr-64k-8n","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MRCR 64K-8N","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":1,"score":17.82,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":2,"score":17.23,"version":"Ministral 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":3,"score":16.56,"version":"MiniCPM SALA"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":4,"score":13.24,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":5,"score":6.93,"version":"Falcon H1R"}],"stars":"","top_company":"qwen","top_model":"Qwen3-8B","top_model_family":"Qwen","top_model_id":"qwen3-8b","top_model_version":"Qwen3","top_open_source":1,"top_percent":17.82,"topic":"Long-context reasoning","total_models":1613,"url":""},{"bench_models":14,"description":"Multi-round multimodal chain-of-reasoning evaluation (v2).","github":"","has_recent":0,"id":"mrcr-v2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MRCR v2","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":1,"score":89.4,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":2,"score":81.7,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":79.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":79,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":5,"score":74.4,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":6,"score":74,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":7,"score":66.9,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":56.2,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":9,"score":55,"version":"Kimi K2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":10,"score":54,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":11,"score":51.4,"version":"Seed 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":12,"score":50.8,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":13,"score":50.1,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":47.1,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":15,"score":33.6,"version":"Seed 2.0"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 High","top_model_family":"GPT","top_model_id":"gpt-5.2-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":89.4,"topic":"Multimodal reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Earth science multiple-choice question benchmark for scientific AI models.","github":"","has_recent":0,"id":"msearth-mcq","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MSEarth-MCQ","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":65.8,"version":"Gemini 3"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":2,"score":65.2,"version":"Intern-S1"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":62.6,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":61.9,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":52.7,"version":"Qwen3 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":65.8,"topic":"Earth science","total_models":1613,"url":""},{"bench_models":46,"description":"Multi-turn chat evaluation via GPT-4 grading.","github":"","has_recent":0,"id":"mt-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MT-Bench","paper":"https://arxiv.org/abs/2306.05685v4","placements":[{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":1,"score":85.7,"version":"Apriel Nemotron"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":2,"score":83.9,"version":"EXAONE 4.0"},{"company":"openai","family":"o1","model":"o1-mini","place":3,"score":83.8,"version":"o1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":4,"score":83.1,"version":"QwQ"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":5,"score":74.2,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":6,"score":9.2,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":7,"score":9.16,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":8,"score":9.09,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":9,"score":9.05,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":10,"score":8.98,"version":"Llama 3.3"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":11,"score":8.98,"version":"Llama 4"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":12,"score":8.85,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":13,"score":8.75,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":14,"score":8.74,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":15,"score":8.73,"version":"Falcon 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":16,"score":8.72,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":17,"score":8.71,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":18,"score":8.69,"version":"Gemma 3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":19,"score":8.53,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":20,"score":8.46,"version":"Falcon H1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":21,"score":8.46,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":22,"score":8.45,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":23,"score":8.45,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":24,"score":8.42,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":25,"score":8.38,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":26,"score":8.3,"version":"Kanana 2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":27,"score":8.29,"version":"Llama 3.1"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":28,"score":8.24,"version":"Gemma 3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":29,"score":8.23,"version":"Kanana 1.5"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":30,"score":7.96,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":31,"score":7.79,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":32,"score":7.79,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":33,"score":7.61,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":34,"score":7.3,"version":"Mistral 3.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":35,"score":7.1,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":36,"score":7.06,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":37,"score":7.03,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":38,"score":6.52,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":39,"score":6.37,"version":"Llama 3.2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":40,"score":6.13,"version":"SmolLM2"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":41,"score":6.03,"version":"Falcon 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":42,"score":5.75,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-1b","place":43,"score":5.48,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":44,"score":4.71,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":45,"score":4.33,"version":"SmolLM1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":46,"score":1.98,"version":"SmolLM2"}],"stars":"39074","top_company":"servicenow","top_model":"Apriel Nemotron 15B Thinker","top_model_family":"Apriel","top_model_id":"apriel-nemotron-15b-thinker","top_model_version":"Apriel Nemotron","top_open_source":1,"top_percent":85.7,"topic":"Chat ability","total_models":1613,"url":"https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge"},{"bench_models":4,"description":"Long-context book understanding benchmark (full-book setting).","github":"","has_recent":0,"id":"mtob-full-book","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MTOB (full book)","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-4-maverick","place":1,"score":50.8,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":2,"score":50.8,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":3,"score":39.7,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":4,"score":39.7,"version":"Llama 4"}],"stars":"","top_company":"meta","top_model":"Llama 4 Maverick","top_model_family":"Llama","top_model_id":"llama-4-maverick","top_model_version":"Llama 4","top_open_source":1,"top_percent":50.8,"topic":"Long-form reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Long-context book understanding benchmark (half-book setting).","github":"","has_recent":0,"id":"mtob-half-book","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MTOB (half book)","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-4-maverick","place":1,"score":54,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e","place":2,"score":54,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":3,"score":42.2,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e","place":4,"score":42.2,"version":"Llama 4"}],"stars":"","top_company":"meta","top_model":"Llama 4 Maverick","top_model_family":"Llama","top_model_id":"llama-4-maverick","top_model_version":"Llama 4","top_open_source":1,"top_percent":54,"topic":"Long-form reasoning","total_models":1613,"url":""},{"bench_models":43,"description":"Evaluates multimodal understanding robustness and reliability.","github":"","has_recent":0,"id":"muirbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MUIRBENCH","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":86.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":2,"score":80.1,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":79.77,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":78.7,"version":"Seed 1.8"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":5,"score":78.6,"version":"dots.vlm1"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":78.6,"version":"GPT-5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":7,"score":78.58,"version":"dots.vlm1"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":8,"score":78.5,"version":"Ovis 2.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":78.2,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":10,"score":78.2,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":77.6,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":12,"score":77.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":77.2,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":14,"score":77.1,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":15,"score":75.7,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":16,"score":75.3,"version":"GLM 4.5V"},{"company":"stepfun","family":"Step","model":"step-3","place":17,"score":75,"version":"Step 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":18,"score":74.7,"version":"GLM 4.1V"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":19,"score":74.7,"version":"GLM 4.1V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":20,"score":74.5,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":21,"score":73.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":22,"score":72.8,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":23,"score":72.3,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":24,"score":71.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":25,"score":71.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":26,"score":71.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":27,"score":70.7,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":28,"score":70.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":29,"score":69.38,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":30,"score":67,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":31,"score":66.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":32,"score":66.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":33,"score":64.4,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":34,"score":63.8,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":35,"score":63.8,"version":"Qwen3 VL"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":36,"score":63.7,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":37,"score":62.9,"version":"Qwen3 VL"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":38,"score":61.8,"version":"Eagle 2.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":39,"score":60.5,"version":"Molmo2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":40,"score":59.6,"version":"Claude 4.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":41,"score":58.4,"version":"Molmo2"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":42,"score":57.5,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":43,"score":55.8,"version":"InternVL 3.5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":44,"score":53.3,"version":"MiniCPM V 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":45,"score":53.1,"version":"InternVL 3.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":46,"score":51.2,"version":"Keye-VL 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":47,"score":50.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":48,"score":47.4,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":49,"score":45.7,"version":"GPT-5"},{"company":"unknown","family":"PLM","model":"plm-3b","place":50,"score":25.7,"version":"PLM"},{"company":"unknown","family":"PLM","model":"plm-8b","place":51,"score":23.5,"version":"PLM"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":86.1,"topic":"Multimodal robustness","total_models":1613,"url":""},{"bench_models":52,"description":"Composite instruction-following evaluation across multiple tasks.","github":"","has_recent":0,"id":"multi-if","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Multi-IF","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":1,"score":81.03,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3-mini-high","place":2,"score":79.5,"version":"o3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":78.2,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":77.93,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":5,"score":77.88,"version":"Kanana 2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":77.8,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":7,"score":77.18,"version":"Ling 2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":8,"score":76.6,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":9,"score":76.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":75.97,"version":"GPT-5.2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":11,"score":75.81,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":12,"score":75.54,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":13,"score":74.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":14,"score":73.2,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b-exp","place":15,"score":73.05,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":16,"score":73,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":17,"score":73,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":18,"score":72.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":19,"score":71.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":20,"score":71.9,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":21,"score":70.8,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":22,"score":70.4,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":23,"score":70.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":24,"score":70.2,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":25,"score":69.93,"version":"Ling 1T"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":26,"score":69.81,"version":"DeepSeek V3.2"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-thinking","place":27,"score":69.33,"version":"LFM2.5"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":28,"score":68.51,"version":"Kanana 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":29,"score":67.9,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":30,"score":67.7,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":31,"score":66.9,"version":"DeepSeek V3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":32,"score":66.61,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":33,"score":66.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-4","place":34,"score":65,"version":"GPT-4"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":35,"score":64.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":36,"score":64,"version":"Qwen3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-1.2b-instruct","place":37,"score":60.98,"version":"LFM2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":38,"score":60.46,"version":"LFM2"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b-thinking","place":39,"score":60.33,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3n-e2b-it","place":40,"score":60.29,"version":"Gemma 3n"},{"company":"liquidai","family":"LFM","model":"lfm2-2.6b","place":41,"score":60.26,"version":"LFM2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":42,"score":58.86,"version":"SmollM3"},{"company":"liquidai","family":"LFM","model":"lfm2-8b-a1b","place":43,"score":58.19,"version":"LFM2"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":44,"score":57.96,"version":"Granite 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":45,"score":56.48,"version":"Qwen3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":46,"score":52.99,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":47,"score":50.93,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-instruct","place":48,"score":50.91,"version":"Llama 3.2"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":49,"score":48.8,"version":"o1"},{"company":"openai","family":"o1","model":"o1","place":50,"score":48.8,"version":"o1"},{"company":"openai","family":"o3","model":"o3-mini","place":51,"score":48.4,"version":"o3"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-1b","place":52,"score":47.56,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":53,"score":44.31,"version":"Gemma 3"},{"company":"IBM","family":"Granite","model":"granite-4.0-1b","place":54,"score":43.65,"version":"Granite 4.0"},{"company":"meta","family":"Llama","model":"llama-3.2.1b-instruct","place":55,"score":30.16,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b-instruct","place":56,"score":30.16,"version":"Llama 3.2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b","top_model_version":"Qwen3","top_open_source":1,"top_percent":81.03,"topic":"Instruction following (multi-task)","total_models":1613,"url":""},{"bench_models":15,"description":"Multi-task variant of instruction-following evaluation.","github":"","has_recent":0,"id":"multi-ifeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Multi-IFEval","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":1,"score":88.7,"version":"Llama 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":2,"score":80.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":3,"score":77.8,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":4,"score":77.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":5,"score":76.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":6,"score":75.8,"version":"Qwen3 Next"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":7,"score":74.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":8,"score":73,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":9,"score":70.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":10,"score":67.9,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-deep-2.4b","place":11,"score":54.5,"version":"EXAONE Deep"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-1.2b","place":12,"score":53.9,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen-3-1.7b","place":13,"score":53.5,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":14,"score":47.5,"version":"SmolLM3"},{"company":"qwen","family":"Qwen","model":"qwen-3-0.6b","place":15,"score":37.5,"version":"Qwen3"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B","top_model_family":"Llama","top_model_id":"llama-3.3-70b","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":88.7,"topic":"Instruction following (multi-task)","total_models":1613,"url":""},{"bench_models":31,"description":"Multi-repository SWE-Bench variant.","github":"","has_recent":0,"id":"multi-swe-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Multi-SWE-Bench","paper":"","placements":[{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":1,"score":51.3,"version":"MiniMax M2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":50.3,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":50,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":4,"score":49.4,"version":"MiniMax M2.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":5,"score":47.2,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":6,"score":44.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":7,"score":44.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":44.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":42.7,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":10,"score":42,"version":"Seed 1.8"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":11,"score":41.9,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":12,"score":41.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":13,"score":39.3,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":14,"score":36.2,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":15,"score":35.7,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":16,"score":35.7,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":17,"score":33.5,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":18,"score":33.5,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":19,"score":33.5,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":20,"score":32.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":21,"score":31.7,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":22,"score":31.3,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":23,"score":30.6,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":24,"score":30.6,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":25,"score":30,"version":"GLM 4.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":26,"score":29.2,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":27,"score":29,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":28,"score":20.7,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":29,"score":18.3,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":30,"score":14,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":31,"score":11.7,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":32,"score":9,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":33,"score":8,"version":"DeepSeek V3"}],"stars":"246","top_company":"minimax","top_model":"MiniMax M2.5","top_model_family":"MiniMax","top_model_id":"minimax-m2.5","top_model_version":"MiniMax M2.5","top_open_source":1,"top_percent":51.3,"topic":"Code repair (multi-repo)","total_models":1613,"url":"https://multi-swe-bench.github.io"},{"bench_models":59,"description":"Multi-domain instruction-following benchmark.","github":"","has_recent":0,"id":"multichallenge","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MultiChallenge","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":69.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":69.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":69.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":68.7,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":5,"score":68.3,"version":"Seed 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":6,"score":67.6,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":67.4,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":8,"score":66.7,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":64.2,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":10,"score":63.3,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":11,"score":63.2,"version":"Seed 2.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":12,"score":62.7,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":13,"score":61.5,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":14,"score":61.1,"version":"Seed 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":15,"score":60.8,"version":"Qwen3.5"},{"company":"openai","family":"o3","model":"o3","place":16,"score":60.4,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":17,"score":60,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":18,"score":59.5,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":19,"score":59,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":20,"score":59,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":21,"score":59,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":22,"score":57.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":23,"score":57.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":24,"score":57.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":25,"score":57.2,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3","place":26,"score":56.5,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":27,"score":55.4,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":28,"score":54.95,"version":"Ling 1T"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":29,"score":54.2,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":30,"score":54.1,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":31,"score":53.11,"version":"Kimi K2.5"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":32,"score":52.21,"version":"Nanbeige4.1"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":33,"score":52.2,"version":"Nanbeige4.1"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":34,"score":52.01,"version":"Ling 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":35,"score":51.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":36,"score":51.28,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":37,"score":50.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":38,"score":49.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-2507","place":39,"score":49.4,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":40,"score":49,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":41,"score":47.3,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":42,"score":46.8,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":43,"score":45.8,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":44,"score":45.3,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":45,"score":45,"version":"DeepSeek R1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":46,"score":44.7,"version":"MiniMax M1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":47,"score":44.7,"version":"MiniMax M1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":48,"score":43.59,"version":"DeepSeek V3.2"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":49,"score":43,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":50,"score":41.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":51,"score":41.76,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":52,"score":41.2,"version":"Nanbeige4"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":53,"score":41.14,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":54,"score":40.7,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":55,"score":40.3,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":56,"score":40,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":57,"score":39.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":58,"score":39.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":59,"score":38.72,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"lg-exaone-32b","place":60,"score":38.5,"version":"EXAONE 4.0"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":61,"score":37.7,"version":"QwQ"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":62,"score":36.97,"version":"Qwen3"},{"company":"servicenow","family":"Apriel","model":"apriel-nemotron-15b-thinker","place":63,"score":36.6,"version":"Apriel Nemotron"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":64,"score":36.4,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":65,"score":36.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":66,"score":36.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":67,"score":35.8,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct-2601","place":68,"score":35.16,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-instruct","place":69,"score":34.8,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":70,"score":34,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":71,"score":31.4,"version":"DeepSeek V3"},{"company":"openai","family":"o1","model":"o1-mini","place":72,"score":30.8,"version":"o1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":73,"score":27.84,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-instruct","place":74,"score":19.05,"version":"Kanana 1.5"},{"company":"nvidia","family":"Llama","model":"llama-3.1-nemotron-nano-8b-v1","place":75,"score":16.1,"version":"Llama 3.1"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":69.6,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":19,"description":"Aggregate score over multi-image visual question answering tasks.","github":"","has_recent":0,"id":"multiimg-qa-avg","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Multi-Image QA Average","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":81.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":72.4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":72.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":4,"score":68.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":5,"score":68.2,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":6,"score":67.4,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":7,"score":59.5,"version":"Claude 4.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":8,"score":57.8,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":9,"score":57.6,"version":"Qwen3 VL"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":10,"score":56.4,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":11,"score":56.3,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":12,"score":54.9,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":13,"score":53.5,"version":"InternVL 3.5"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":14,"score":53.5,"version":"Molmo2"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":15,"score":52.1,"version":"Keye-VL 1.5"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":16,"score":52,"version":"Eagle 2.5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":17,"score":47.3,"version":"MiniCPM V 4.5"},{"company":"unknown","family":"PLM","model":"plm-3b","place":18,"score":40.6,"version":"PLM"},{"company":"unknown","family":"PLM","model":"plm-8b","place":19,"score":35.7,"version":"PLM"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":81.9,"topic":"Multi-image QA (aggregate)","total_models":1613,"url":""},{"bench_models":4,"description":"Multilingual MMBench average score across languages.","github":"","has_recent":0,"id":"multilingual-mmbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Multilingual MMBench","paper":"","placements":[{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":1,"score":65.9,"version":"LFM2.5-VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":2,"score":60.57,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":3,"score":58.32,"version":"InternVL 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":4,"score":50.89,"version":"FastVLM"}],"stars":"","top_company":"liquidai","top_model":"LFM2.5-VL-1.6B","top_model_family":"LFM","top_model_id":"lfm2.5-vl-1.6b","top_model_version":"LFM2.5-VL","top_open_source":1,"top_percent":65.9,"topic":"Multilingual vision benchmark","total_models":1613,"url":""},{"bench_models":12,"description":"Multilingual variant of MMLU across many languages.","github":"","has_recent":0,"id":"multilingual-mmlu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Multilingual MMLU","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-4.1","place":1,"score":87.3,"version":"GPT-4.1"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":2,"score":86.24,"version":"Cogito 671B v2.1"},{"company":"meta","family":"Llama","model":"llama-4-behemoth","place":3,"score":85.8,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":4,"score":85.1,"version":"GPT-4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-base","place":5,"score":75.4,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-14b-base","place":6,"score":74.2,"version":"Ministral 3"},{"company":"mistral","family":"Ministral","model":"ministral-3-8b-base","place":7,"score":70.6,"version":"Ministral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-base","place":8,"score":70,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-12b-base","place":9,"score":69,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-base","place":10,"score":67.7,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":11,"score":65.2,"version":"Ministral 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-base","place":12,"score":51.6,"version":"Gemma 3"}],"stars":"","top_company":"openai","top_model":"GPT-4.1","top_model_family":"GPT","top_model_id":"gpt-4.1","top_model_version":"GPT-4.1","top_open_source":0,"top_percent":87.3,"topic":"Multi-domain knowledge (multilingual)","total_models":1613,"url":""},{"bench_models":41,"description":"Multilingual code generation and execution benchmark across many programming languages.","github":"","has_recent":0,"id":"multipl-e","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MultiPL-E","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":1,"score":89.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":2,"score":88.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":3,"score":87.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":4,"score":87.8,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":5,"score":86.7,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":6,"score":85.7,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":7,"score":85.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":8,"score":84.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":9,"score":83.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":10,"score":83.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":11,"score":83.1,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":12,"score":82.7,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":13,"score":81.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":14,"score":81.4,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":15,"score":81.4,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":16,"score":81.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":17,"score":81,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":18,"score":80.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":19,"score":79.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":20,"score":79.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":21,"score":78.2,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":22,"score":77.91,"version":"Ling 1T"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":23,"score":77.68,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":24,"score":76.9,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5","place":25,"score":76.66,"version":"GPT-5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":26,"score":74.87,"version":"LLaDA 2.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":27,"score":73.54,"version":"Kimi K2"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":28,"score":73.34,"version":"LLaDA 2.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":29,"score":71.48,"version":"Gemini 2.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":30,"score":70.89,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":31,"score":70.67,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":32,"score":67.46,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":33,"score":67.17,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":34,"score":67.09,"version":"Ling 2.0"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":35,"score":65.76,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":36,"score":64.16,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":37,"score":62.43,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":38,"score":61.7,"version":"Qwen3"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-7b-pretrain","place":39,"score":60.41,"version":"Hunyuan"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":40,"score":57.37,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":41,"score":49.21,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":42,"score":48.91,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":43,"score":47.26,"version":"Granite 4.0"}],"stars":"269","top_company":"anthropic","top_model":"Claude Opus 4","top_model_family":"Claude","top_model_id":"claude-opus-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":89.6,"topic":"Code generation (multilingual)","total_models":1613,"url":"https://github.com/nuprl/MultiPL-E"},{"bench_models":7,"description":"MultiPL-E variant of HumanEval tasks.","github":"","has_recent":0,"id":"multipl-e-humaneval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MultiPL-E HumanEval","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":1,"score":75.2,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":2,"score":65.5,"version":"Llama 3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":3,"score":60.5,"version":"Kimi K2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":4,"score":59.5,"version":"MiMo V2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":5,"score":50.8,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":6,"score":45.9,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":7,"score":45.7,"version":"DeepSeek V3.2"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B","top_model_family":"Llama","top_model_id":"llama-3.1-405b","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":75.2,"topic":"Code generation (multilingual)","total_models":1613,"url":"https://github.com/nuprl/MultiPL-E"},{"bench_models":7,"description":"MultiPL-E variant of MBPP tasks.","github":"","has_recent":0,"id":"multipl-e-mbpp","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MultiPL-E MBPP","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":1,"score":65.7,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":2,"score":62,"version":"Llama 3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":3,"score":58.8,"version":"Kimi K2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":4,"score":56.7,"version":"MiMo V2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":5,"score":52.5,"version":"DeepSeek V3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":6,"score":52.4,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":7,"score":50.6,"version":"DeepSeek V3.2"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B","top_model_family":"Llama","top_model_id":"llama-3.1-405b","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":65.7,"topic":"Code generation (multilingual)","total_models":1613,"url":"https://github.com/nuprl/MultiPL-E"},{"bench_models":49,"description":"Multistep Soft Reasoning.","github":"","has_recent":0,"id":"musr","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MuSR","paper":"https://arxiv.org/abs/2310.16049","placements":[{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":1,"score":82.72,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":2,"score":82.72,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":3,"score":80.1,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":4,"score":79.84,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":5,"score":79.15,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":6,"score":71.89,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":7,"score":71.48,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":8,"score":71.36,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":9,"score":70.43,"version":"LLaDA 2.1"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-70b","place":10,"score":70.4,"version":"Hermes 4.3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":11,"score":70.02,"version":"Qwen3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":12,"score":69.9,"version":"ERNIE 4.5"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b-pyche","place":13,"score":69.7,"version":"Hermes 4.3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":14,"score":66.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":15,"score":65.9,"version":"DeepSeek V3"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b","place":16,"score":64.7,"version":"Hermes 4.3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":17,"score":64.47,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":18,"score":64.4,"version":"Qwen3"},{"company":"arcee","family":"Trinity","model":"trinity-mini","place":19,"score":63.49,"version":"Trinity"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":20,"score":62.6,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":21,"score":60.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":22,"score":59.7,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":23,"score":58.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":24,"score":57.7,"version":"Qwen3"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":25,"score":57.4,"version":"Youtu-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":26,"score":56.6,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-2509","place":27,"score":56.3,"version":"Magistral 1.2"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":28,"score":54.1,"version":"SmollM3"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-base-instruct","place":29,"score":51.9,"version":"Mi:dm 2.0"},{"company":"kt","family":"Mi:dm","model":"midm-2.0-mini-instruct","place":30,"score":51.7,"version":"Mi:dm 2.0"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-7.8b-instruct","place":31,"score":51.2,"version":"EXAONE 3.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":32,"score":50.3,"version":"Llama 3.1"},{"company":"lg","family":"EXAONE","model":"exaone-3.5-2.4b-instruct","place":33,"score":49.7,"version":"EXAONE 3.5"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":34,"score":47.22,"version":"Llama 3.1"},{"company":"microsoft","family":"Phi","model":"phi-3.5-moe-instruct","place":35,"score":46.79,"version":"Phi 3.5 MoE"},{"company":"upstage","family":"Solar","model":"solar-pro-preview","place":36,"score":45.85,"version":"Solar Pro"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":37,"score":45.67,"version":"Gemma 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-1.5b","place":38,"score":43.8,"version":"DeepSeek R1"},{"company":"cohere","family":"Command","model":"command-r7b","place":39,"score":43.3,"version":"Command"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":40,"score":42.28,"version":"Phi 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":41,"score":41.14,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":42,"score":39.95,"version":"Gemma 3"},{"company":"arcee","family":"AFM","model":"afm-4.5b","place":43,"score":38.76,"version":"AFM"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":44,"score":36.77,"version":"SmollM3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":45,"score":29.68,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":46,"score":13.65,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":47,"score":9.71,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":48,"score":7.73,"version":"SmolLM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":49,"score":5.27,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":50,"score":4.6,"version":"SmolLM2"},{"company":"tii","family":"Falcon","model":"falcon-e-1b-base","place":51,"score":3.65,"version":"Falcon E"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":52,"score":3.38,"version":"Qwen3"},{"company":"arcee","family":"Llama Spark","model":"llama-spark-8b","place":53,"score":2.62,"version":"Llama Spark 8B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":54,"score":2.08,"version":"Qwen2.5"}],"stars":"","top_company":"antgroup","top_model":"Ling Flash 2.0","top_model_family":"Ling","top_model_id":"ling-flash-2.0","top_model_version":"Ling 2.0","top_open_source":1,"top_percent":82.72,"topic":"Reasoning","total_models":1613,"url":""},{"bench_models":26,"description":"Multi-view or multi-video QA benchmark (MVBench).","github":"","has_recent":0,"id":"mvbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MVBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":78.1,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":2,"score":77.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":76.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":4,"score":75.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":5,"score":74.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":6,"score":74.6,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":74.1,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":8,"score":73.5,"version":"Kimi K2.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":9,"score":73,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":10,"score":72.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":72.3,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":12,"score":71.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":13,"score":71,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":14,"score":70.4,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":15,"score":68.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":16,"score":68.7,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":17,"score":68.4,"version":"GLM 4.1V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":18,"score":67.2,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":19,"score":65,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":20,"score":61.7,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":21,"score":59.7,"version":"Kimi VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":22,"score":49.4,"version":"GPT-5"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":23,"score":46.27,"version":"SmolVLM2"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":24,"score":43.5,"version":"Gemma 3"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-0.5b","place":25,"score":39.73,"version":"SmolVLM2"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-0.256b","place":26,"score":32.7,"version":"SmolVLM2"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":78.1,"topic":"Video QA","total_models":1613,"url":""},{"bench_models":8,"description":"Natural language to code benchmark for instruction-following synthesis.","github":"","has_recent":0,"id":"natural2code","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Natural2Code","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":1,"score":92.9,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":2,"score":85.4,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":3,"score":84.5,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":4,"score":80.7,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":5,"score":79.8,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-001","place":6,"score":75.5,"version":"Gemini 1.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":7,"score":70.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":8,"score":56,"version":"Gemma 3"}],"stars":"","top_company":"google","top_model":"Gemini 2.0 Flash","top_model_family":"Gemini","top_model_id":"gemini-2.0-flash","top_model_version":"Gemini 2.0","top_open_source":0,"top_percent":92.9,"topic":"Code generation","total_models":1613,"url":""},{"bench_models":17,"description":"Google NQ; real user questions with long/short answers.","github":"","has_recent":0,"id":"naturalquestions","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NaturalQuestions","paper":"","placements":[{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":1,"score":40.1,"version":"Mixtral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":2,"score":36.5,"version":"Mixtral"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":3,"score":36.1,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":4,"score":36.1,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":5,"score":35.5,"version":"Llama 2"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":6,"score":31.4,"version":"Gemma 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":7,"score":30.6,"version":"Mixtral"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":8,"score":29.6,"version":"Motif 2"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":9,"score":29,"version":"Llama 2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":10,"score":28.8,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":11,"score":28.1,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":12,"score":25.4,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":13,"score":24.7,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":14,"score":24.1,"version":"Llama 1"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":15,"score":23.2,"version":"Mistral"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":16,"score":23,"version":"Gemma 2"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":17,"score":17.5,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":18,"score":16.7,"version":"Llama 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":19,"score":15.76,"version":"MobileLLM P1"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":20,"score":12.5,"version":"Gemma 2"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":21,"score":11,"version":"CodeLlama-7B"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":22,"score":9.48,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":23,"score":5.48,"version":"Llama 3.2"}],"stars":"","top_company":"mistral","top_model":"Mixtral 8x22B","top_model_family":"Mixtral","top_model_id":"mixtral-8x22b","top_model_version":"Mixtral","top_open_source":1,"top_percent":40.1,"topic":"Open-domain QA","total_models":1613,"url":"https://ai.google.com/research/NaturalQuestions"},{"bench_models":15,"description":"Nexus tool-use benchmark, zero-shot setting.","github":"","has_recent":0,"id":"nexus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Nexus (0-shot)","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b","place":1,"score":58.7,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b","place":2,"score":56.7,"version":"Llama 3.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":3,"score":50.45,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":49.93,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":5,"score":47.8,"version":"Llama 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":6,"score":47.65,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":7,"score":44.83,"version":"LLaDA 2.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":8,"score":38.5,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":9,"score":37.71,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":10,"score":36.25,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":11,"score":35.18,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":12,"score":34.38,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":13,"score":33.69,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":14,"score":31.59,"version":"LLaDA 2.1"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":15,"score":18.1,"version":"Llama 3"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B","top_model_family":"Llama","top_model_id":"llama-3.1-405b","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":58.7,"topic":"Tool use","total_models":1613,"url":""},{"bench_models":3,"description":"Needle In A Haystack test for locating hidden facts in long contexts.","github":"https://github.com/gkamradt/LLMTest_NeedleInAHaystack","has_recent":0,"id":"nih","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Needle In A Haystack","paper":"","placements":[{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":1,"score":100,"version":"MobileLLM P1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":2,"score":98.67,"version":"Llama 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":3,"score":96.44,"version":"Gemma 3"}],"stars":"","top_company":"meta","top_model":"MobileLLM P1 Base","top_model_family":"MobileLLM","top_model_id":"mobilellm-p1-base","top_model_version":"MobileLLM P1","top_open_source":1,"top_percent":100,"topic":"Long-context retrieval","total_models":1613,"url":"https://github.com/gkamradt/LLMTest_NeedleInAHaystack"},{"bench_models":5,"description":"NoLiMa (No Literal Match) long-context benchmark at 128k context window.","github":"","has_recent":0,"id":"nolima-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NoLiMa 128K","paper":"","placements":[{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":1,"score":23.86,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":2,"score":11.25,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":3,"score":5.8,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":4,"score":4.73,"version":"Falcon H1R"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":5,"score":3.48,"version":"Ministral 3"}],"stars":"","top_company":"openbmb","top_model":"MiniCPM-SALA","top_model_family":"MiniCPM","top_model_id":"minicpm-sala","top_model_version":"MiniCPM SALA","top_open_source":1,"top_percent":23.86,"topic":"Long-context eval","total_models":1613,"url":""},{"bench_models":5,"description":"NoLiMa (No Literal Match) long-context benchmark at 32k context window.","github":"","has_recent":0,"id":"nolima-32k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NoLiMa 32K","paper":"","placements":[{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":1,"score":54.54,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":2,"score":43.4,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":3,"score":19.69,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":4,"score":14.89,"version":"Falcon H1R"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":5,"score":3.78,"version":"Ministral 3"}],"stars":"","top_company":"openbmb","top_model":"MiniCPM-SALA","top_model_family":"MiniCPM","top_model_id":"minicpm-sala","top_model_version":"MiniCPM SALA","top_open_source":1,"top_percent":54.54,"topic":"Long-context eval","total_models":1613,"url":""},{"bench_models":5,"description":"NoLiMa (No Literal Match) long-context benchmark at 64k context window.","github":"","has_recent":0,"id":"nolima-64k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NoLiMa 64K","paper":"","placements":[{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":1,"score":42.95,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":2,"score":23.35,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":3,"score":11.82,"version":"Nemotron Nano V2"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":4,"score":9.87,"version":"Falcon H1R"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":5,"score":2.48,"version":"Ministral 3"}],"stars":"","top_company":"openbmb","top_model":"MiniCPM-SALA","top_model_family":"MiniCPM","top_model_id":"minicpm-sala","top_model_version":"MiniCPM SALA","top_open_source":1,"top_percent":42.95,"topic":"Long-context eval","total_models":1613,"url":""},{"bench_models":12,"description":"Multilingual evaluation benchmark covering 63 languages.","github":"","has_recent":0,"id":"nova-63","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NOVA-63","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":59.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":2,"score":58.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":3,"score":58.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":4,"score":57.1,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":56.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":56.7,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":56,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":8,"score":55.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":9,"score":54.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":10,"score":54.2,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":11,"score":51.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":12,"score":51.1,"version":"GPT-OSS"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":59.1,"topic":"Multilingual evaluation","total_models":1613,"url":""},{"bench_models":5,"description":"3D scene understanding and perception benchmark for autonomous driving.","github":"","has_recent":0,"id":"nuscenes","leaderboard":"","lower_is_better":false,"metric":"percent","name":"NuScenes","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":16,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":2,"score":15.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":3,"score":15.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":4,"score":14.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":5,"score":13.9,"version":"Qwen3 VL"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":16,"topic":"3D scene understanding","total_models":1613,"url":""},{"bench_models":3,"description":"Objectron benchmark for 3D object detection in video captures.","github":"","has_recent":0,"id":"objectron","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Objectron","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":1,"score":71.2,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":2,"score":8.1,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":5.5,"version":"Gemini 2.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-235B-A22B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-235b-a22b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":71.2,"topic":"Object detection","total_models":1613,"url":""},{"bench_models":5,"description":"OpenBookQA science question answering benchmark.","github":"","has_recent":0,"id":"obqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OBQA","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":1,"score":76.26,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":2,"score":44.4,"version":"LFM2.5-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":3,"score":44.4,"version":"LFM2-Audio"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":4,"score":26.59,"version":"Mini-Omni2"},{"company":"kyutai","family":"Moshi","model":"moshi","place":5,"score":25.93,"version":"Moshi"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5-Omni-3B","top_model_family":"Qwen","top_model_id":"qwen2.5-omni-3b","top_model_version":"Qwen2.5 Omni","top_open_source":1,"top_percent":76.26,"topic":"Open book QA","total_models":1613,"url":""},{"bench_models":10,"description":"Original Chinese Natural Language Inference benchmark.","github":"","has_recent":0,"id":"ocnli","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OCNLI","paper":"","placements":[{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":1,"score":72.75,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":2,"score":72.17,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":3,"score":71.63,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":71.59,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":5,"score":65.36,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":6,"score":64.51,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":7,"score":61.59,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":8,"score":61.49,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":9,"score":61.02,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":10,"score":60.17,"version":"Ling 2.0"}],"stars":"","top_company":"antgroup","top_model":"LLaDA2.1-Flash (Q Mode)","top_model_family":"LLaDA","top_model_id":"llada2.1-flash-q","top_model_version":"LLaDA 2.1","top_open_source":1,"top_percent":72.75,"topic":"Natural language inference (Chinese)","total_models":1613,"url":""},{"bench_models":84,"description":"OCRBench v2 evaluating text extraction from images and documents.","github":"","has_recent":0,"id":"ocrbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OCRBench V2","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":1,"score":858,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":2,"score":836,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":3,"score":835,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":4,"score":809,"version":"Qwen2 VL"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":5,"score":778,"version":"jina-VLM"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":6,"score":93.1,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":7,"score":92.3,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":8,"score":92.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":9,"score":92.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":10,"score":91,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":11,"score":90.4,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":12,"score":90.3,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":90.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":14,"score":89.7,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":15,"score":89.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":16,"score":89.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":17,"score":89.4,"version":"Qwen3.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":18,"score":89.3,"version":"Ovis 2.6"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-pacore","place":19,"score":89,"version":"Step3-VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":20,"score":88.5,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":21,"score":88.1,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":22,"score":88,"version":"InternVL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":23,"score":87.6,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":24,"score":87.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":25,"score":87.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":26,"score":87.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":27,"score":87.1,"version":"Qwen2.5 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":28,"score":86.9,"version":"Kimi VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b-sere","place":29,"score":86.75,"version":"Step3-VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":30,"score":86.75,"version":"Step3-VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":31,"score":86.7,"version":"Seed 1.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":32,"score":86.6,"version":"MiMo VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":33,"score":86.6,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":34,"score":86.5,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":35,"score":86.5,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":36,"score":86.5,"version":"Claude 4.5"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":37,"score":86.3,"version":"Manzano"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":38,"score":86.2,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":39,"score":86.2,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":40,"score":85.97,"version":"GLM 4.6V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":41,"score":85.9,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":42,"score":85.8,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":43,"score":85.8,"version":"Claude 4.5"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":44,"score":85.7,"version":"Manzano"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":45,"score":85.4,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":46,"score":85.2,"version":"InternVL 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":47,"score":85.2,"version":"Seed 1.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":48,"score":84.7,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-4o","place":49,"score":84.3,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":50,"score":84.2,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":51,"score":83.7,"version":"Step 3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":52,"score":83.7,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":53,"score":83.6,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":54,"score":82.85,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":55,"score":82.8,"version":"InternVL 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":56,"score":82.8,"version":"GPT-5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":57,"score":82.3,"version":"dots.vlm1"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":58,"score":82.2,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":59,"score":82.1,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":60,"score":81.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":61,"score":80.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":62,"score":80.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":63,"score":80.7,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":64,"score":80.6,"version":"Claude 3.7"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":65,"score":80.4,"version":"InternVL 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":66,"score":79.7,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":67,"score":77.6,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":68,"score":76.6,"version":"Claude 4.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":69,"score":75.9,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":70,"score":75.4,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":71,"score":74.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":72,"score":74.1,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":73,"score":73.6,"version":"GPT-4o"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":74,"score":73.3,"version":"Bagel"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":75,"score":73.1,"version":"FastVLM"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":76,"score":70.4,"version":"Qwen2.5 Omni"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":77,"score":70.2,"version":"FastVLM"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":78,"score":70.1,"version":"GPT-5"},{"company":"baai","family":"Emu","model":"emu3-8b","place":79,"score":68.7,"version":"Emu3"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":80,"score":65.8,"version":"MM1.5"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":81,"score":65.7,"version":"MM1.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":82,"score":63.9,"version":"FastVLM"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":83,"score":63.7,"version":"Phi-3"},{"company":"apple","family":"MM1","model":"mm1-30b","place":84,"score":60.6,"version":"MM1"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":85,"score":60.5,"version":"MiniCPM V 2.0"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":86,"score":60,"version":"Cambrian"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":87,"score":59,"version":"Janus Pro"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":88,"score":48.7,"version":"Janus Pro"},{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":89,"score":41.44,"version":"LFM2.5-VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":90,"score":35.11,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":91,"score":33.53,"version":"InternVL 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":92,"score":26.61,"version":"FastVLM"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":93,"score":0,"version":"VILA 1.5"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":94,"score":0,"version":"Gemini Nano"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":95,"score":0,"version":"BLIP-3O"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL 2B Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-2b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":858,"topic":"OCR (vision text extraction)","total_models":1613,"url":""},{"bench_models":8,"description":"OCR benchmark using ELO rating system to rank model performance on text extraction tasks.","github":"","has_recent":0,"id":"ocrbench-elo","leaderboard":"","lower_is_better":false,"metric":"elo","name":"OCRBench-ELO","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":866,"version":"Gemini 2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":2,"score":858,"version":"ERNIE 4.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":3,"score":834,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":4,"score":831,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":5,"score":824,"version":"Qwen2.5 VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":6,"score":822,"version":"LFM2-VL"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":810,"version":"GPT-5"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":8,"score":725,"version":"SmolVLM2"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":866,"topic":"OCR (ELO ranking)","total_models":1613,"url":""},{"bench_models":23,"description":"OCRBenchV2 Chinese subset assessing OCR performance on Chinese-language documents.","github":"","has_recent":0,"id":"ocrbenchv2-cn","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OCRBenchV2 (CN)","paper":"","placements":[{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":1,"score":67.1,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":2,"score":63.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":3,"score":63.7,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":4,"score":63.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":5,"score":63.2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":6,"score":61.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":7,"score":61.2,"version":"Qwen3 VL"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":8,"score":60.6,"version":"Intern-S1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":9,"score":60.4,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":10,"score":59.6,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":11,"score":59.5,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":12,"score":59.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":13,"score":57.6,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":14,"score":57.6,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":15,"score":57.4,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":16,"score":54.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":17,"score":53,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":18,"score":52.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":19,"score":48.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":20,"score":45.1,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":21,"score":43.7,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5","place":22,"score":43.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":23,"score":36.2,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":24,"score":36.2,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":25,"score":34.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":26,"score":27.3,"version":"GPT-5"}],"stars":"","top_company":"alibaba","top_model":"Ovis2.6-30B-A3B","top_model_family":"Ovis","top_model_id":"ovis2.6-30b-a3b","top_model_version":"Ovis 2.6","top_open_source":1,"top_percent":67.1,"topic":"OCR (Chinese)","total_models":1613,"url":""},{"bench_models":23,"description":"OCRBenchV2 English subset evaluating OCR accuracy on English documents and layouts.","github":"","has_recent":0,"id":"ocrbenchv2-en","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OCRBenchV2 (EN)","paper":"","placements":[{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":1,"score":72.6,"version":"Ovis 2.6"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":68,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":3,"score":67.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":4,"score":66.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":5,"score":65.4,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":6,"score":65.1,"version":"GLM 4.6V"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":64.2,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":8,"score":63.7,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":9,"score":63.5,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":10,"score":63.2,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":61.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":12,"score":61.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":13,"score":61.5,"version":"Qwen2.5 VL"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":14,"score":60.1,"version":"Intern-S1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":15,"score":57.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":16,"score":56.4,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":17,"score":56.3,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":54.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":19,"score":53,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":20,"score":52.6,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":21,"score":48.4,"version":"Claude 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":22,"score":48.1,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":23,"score":44.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":24,"score":44.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":25,"score":37.9,"version":"GPT-5"}],"stars":"","top_company":"alibaba","top_model":"Ovis2.6-30B-A3B","top_model_family":"Ovis","top_model_id":"ovis2.6-30b-a3b","top_model_version":"Ovis 2.6","top_open_source":1,"top_percent":72.6,"topic":"OCR (English)","total_models":1613,"url":""},{"bench_models":4,"description":"OCR reasoning benchmark combining text extraction with multi-step reasoning over documents.","github":"","has_recent":0,"id":"ocrreasoning","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OCRReasoning","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":70.81,"version":"Gemini 2.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":2,"score":66.23,"version":"dots.vlm1"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":63.42,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":4,"score":38.02,"version":"Qwen2.5 VL"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":70.81,"topic":"OCR reasoning","total_models":1613,"url":""},{"bench_models":6,"description":"Coding benchmark across multi-language programming tasks.","github":"","has_recent":0,"id":"octocodingbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OctoCodingBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":36.2,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":2,"score":26.1,"version":"MiniMax M2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":3,"score":26,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":22.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":22.8,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":6,"score":13.3,"version":"MiniMax M2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":36.2,"topic":"Code generation","total_models":1613,"url":""},{"bench_models":18,"description":"Object Detection in the Wild benchmark covering 13 real-world domains.","github":"","has_recent":0,"id":"odinw-13","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ODinW-13","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":1,"score":48.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":2,"score":47.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":47,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":4,"score":46.6,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":46.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":6,"score":44.7,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":7,"score":44.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":8,"score":44.5,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":9,"score":43.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":10,"score":43.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":11,"score":43.2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":12,"score":43.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":13,"score":43.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":14,"score":43.1,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":15,"score":42.6,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":16,"score":41.3,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":17,"score":41.1,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":33.7,"version":"Gemini 2.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-4B-Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-4b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":48.2,"topic":"Object detection (in the wild)","total_models":1613,"url":""},{"bench_models":6,"description":"Odyssey multi-step math benchmark.","github":"","has_recent":0,"id":"odyssey-math","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Odyssey Math","paper":"","placements":[{"company":"mistral","family":"Mathstral","model":"mathstral-7b","place":1,"score":37.2,"version":"Mathstral"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":2,"score":27.6,"version":"DeepSeek Math"},{"company":"qwen","family":"Qwen","model":"qwen2-7b","place":3,"score":24.8,"version":"Qwen2"},{"company":"meta","family":"Llama","model":"llama3-8b","place":4,"score":24,"version":"Llama 3"},{"company":"zai","family":"GLM","model":"glm4-9b","place":5,"score":18.9,"version":"GLM 4"},{"company":"google","family":"Gemma 2","model":"gemma2-9b","place":6,"score":18.6,"version":"Gemma 2"}],"stars":"","top_company":"mistral","top_model":"Mathstral 7B","top_model_family":"Mathstral","top_model_id":"mathstral-7b","top_model_version":"Mathstral","top_open_source":1,"top_percent":37.2,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":7,"description":"English subset of OIBench for code generation.","github":"","has_recent":0,"id":"oibench-en","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OIBench EN","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":58.2,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":2,"score":50,"version":"Claude 4.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":3,"score":47.7,"version":"LongCat-Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":43.3,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":5,"score":39,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":6,"score":36.8,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":7,"score":30.8,"version":"GLM 4.7"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":58.2,"topic":"Code generation","total_models":1613,"url":""},{"bench_models":36,"description":"Programming problems evaluated via online judge-style execution.","github":"","has_recent":0,"id":"ojbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OJBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":68.5,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":61.2,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":3,"score":57.4,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":56.2,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":5,"score":54.7,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":6,"score":54.6,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":7,"score":48.7,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":8,"score":46.7,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":9,"score":44.6,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":10,"score":42.3,"version":"Kimi K2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":11,"score":42.2,"version":"LongCat-Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":12,"score":41.8,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":41.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":14,"score":41.5,"version":"GPT-OSS"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking","place":15,"score":40.7,"version":"LongCat-Flash"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":16,"score":40.4,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":17,"score":40.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":18,"score":39.5,"version":"Qwen3.5"},{"company":"openai","family":"o3","model":"o3","place":19,"score":38.4,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":20,"score":38.2,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":21,"score":36,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":22,"score":35.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":23,"score":34.1,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":24,"score":33.6,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":25,"score":32.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":26,"score":32.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking-2507","place":27,"score":32.1,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":28,"score":30.4,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":29,"score":29.7,"version":"Qwen3 Next"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":30,"score":27.1,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":31,"score":25.5,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":32,"score":25.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":33,"score":24.1,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":34,"score":24,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":35,"score":23.5,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":36,"score":19.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":37,"score":19.5,"version":"GPT-4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":38,"score":19.5,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":39,"score":19,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":40,"score":15.3,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":41,"score":14.2,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":42,"score":11.3,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":68.5,"topic":"Code generation (online judge)","total_models":1613,"url":""},{"bench_models":22,"description":"olmOCR benchmark assessing OCR fidelity and structured extraction on complex document pages.","github":"","has_recent":0,"id":"olmocr-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"olmOCR-Bench","paper":"","placements":[{"company":"chandra","family":"Chandra OCR","model":"chandra-ocr-0.1.0","place":1,"score":83.1,"version":"Chandra OCR"},{"company":"infly","family":"Infinity-Parser","model":"infinity-parser-7b","place":2,"score":82.5,"version":"Infinity-Parser"},{"company":"allenai","family":"OLMOCR","model":"olmocr-2-7b-1025","place":3,"score":82.4,"version":"OLMOCR 2"},{"company":"paddlepaddle","family":"PaddleOCR","model":"paddleocr-vl","place":4,"score":80,"version":"PaddleOCR"},{"company":"rednote","family":"dots","model":"dots.ocr","place":5,"score":79.1,"version":"dots"},{"company":"datalab","family":"Marker","model":"marker","place":6,"score":76.1,"version":"Marker"},{"company":"kingsoft","family":"MonkeyOCR","model":"monkeyocr-pro-3b","place":7,"score":75.8,"version":"MonkeyOCR Pro"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-base","place":8,"score":75.7,"version":"DeepSeek OCR"},{"company":"allenai","family":"OLMOCR","model":"olmocr","place":9,"score":75.5,"version":"OLMOCR"},{"company":"opendatalab","family":"MinerU","model":"mineru-2.5.4","place":10,"score":75.2,"version":"MinerU"},{"company":"mistral","family":"Mistral OCR","model":"mistral-ocr-api","place":11,"score":72,"version":"Mistral OCR"},{"company":"datalab","family":"Marker","model":"marker-v1.7.5","place":12,"score":70.1,"version":"Marker"},{"company":"openai","family":"GPT","model":"gpt-4o","place":13,"score":69.9,"version":"GPT-4o"},{"company":"nanonets","family":"Nanonets OCR","model":"nanonets-ocr2-3b","place":14,"score":69.5,"version":"Nanonets OCR2"},{"company":"openai","family":"GPT","model":"gpt-4o","place":15,"score":68.9,"version":"GPT-4o"},{"company":"allenai","family":"OLMOCR","model":"olmocr","place":16,"score":68.2,"version":"OLMOCR"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":17,"score":65.5,"version":"Qwen2.5 VL"},{"company":"nanonets","family":"Nanonets OCR","model":"nanonets-ocr-s","place":18,"score":64.5,"version":"Nanonets OCR"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":19,"score":63.8,"version":"Gemini 2.0"},{"company":"opendatalab","family":"MinerU","model":"mineru-1.3.10","place":20,"score":61.5,"version":"MinerU"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":21,"score":57.8,"version":"Gemini 2.5"},{"company":"stepfun","family":"GOT-OCR","model":"got-ocr","place":22,"score":48.3,"version":"GOT-OCR"},{"company":"qwen","family":"Qwen","model":"qwen2-vl","place":23,"score":31.5,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":24,"score":31.5,"version":"Qwen2 VL"}],"stars":"","top_company":"chandra","top_model":"Chandra OCR 0.1.0","top_model_family":"Chandra OCR","top_model_id":"chandra-ocr-0.1.0","top_model_version":"Chandra OCR","top_open_source":1,"top_percent":83.1,"topic":"Document OCR","total_models":1613,"url":""},{"bench_models":16,"description":"Advanced mathematics olympiad-style problem benchmark.","github":"","has_recent":0,"id":"olympiadbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OlympiadBench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":1,"score":77.59,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":2,"score":76.59,"version":"LLaDA 2.1"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-7b-instruct","place":3,"score":76.5,"version":"Hunyuan"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":4,"score":76.19,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":5,"score":75.85,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":6,"score":74.07,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":7,"score":72.3,"version":"Ling 2.0"},{"company":"amazon","family":"Nemotron","model":"questa-nemotron-1.5b","place":8,"score":70.36,"version":"QuestA Nemotron"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":9,"score":67.7,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":10,"score":66.67,"version":"LLaDA 2.1"},{"company":"deepseek","family":"DeepSeek-R1","model":"deepseek-r1-distill-32b","place":11,"score":65,"version":"32B"},{"company":"nvidia","family":"Nemotron","model":"nemotron-1.5b","place":12,"score":64.62,"version":"Nemotron"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":13,"score":64.3,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":14,"score":56.13,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":15,"score":55.33,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":16,"score":47.78,"version":"LLaDA 2.0"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Instruct-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-instruct-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":77.59,"topic":"Math (olympiad)","total_models":1613,"url":""},{"bench_models":4,"description":"Olympiad-style mathematics reasoning benchmark.","github":"","has_recent":0,"id":"olympic-arena","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OlympicArena","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":1,"score":76.2,"version":"DeepSeek V3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":2,"score":75.3,"version":"ERNIE 4.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":3,"score":69.5,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":4,"score":63.7,"version":"Qwen3"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3","top_model_family":"DeepSeek","top_model_id":"deepseek-v3","top_model_version":"DeepSeek V3","top_open_source":1,"top_percent":76.2,"topic":"Math (competition)","total_models":1613,"url":""},{"bench_models":10,"description":"OMEGA olympiad-grade mathematics reasoning benchmark.","github":"","has_recent":0,"id":"omega","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OMEGA","paper":"","placements":[{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":1,"score":50.8,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":2,"score":50.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":3,"score":47.7,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":4,"score":45.2,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":5,"score":43.1,"version":"OLMo 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":6,"score":38.9,"version":"DeepSeek R1"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":7,"score":24,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":8,"score":19.2,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":9,"score":9.8,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":10,"score":9.1,"version":"Gemma 2"}],"stars":"","top_company":"allenai","top_model":"OLMo-3-Think-32B","top_model_family":"OLMo","top_model_id":"olmo-3-think-32b","top_model_version":"OLMo 3","top_open_source":1,"top_percent":50.8,"topic":"Math (advanced)","total_models":1613,"url":""},{"bench_models":22,"description":"Omni-MATH benchmark covering diverse math reasoning tasks across difficulty levels.","github":"","has_recent":0,"id":"omni-math","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Omni-MATH","paper":"","placements":[{"company":"antgroup","family":"Ling","model":"ling-1t","place":1,"score":74.46,"version":"Ling 1T"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":72.02,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":3,"score":64.77,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":4,"score":62.42,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":5,"score":61.7,"version":"Qwen3 Next"},{"company":"bytedance","family":"Seed","model":"seed-oss-36b-instruct","place":6,"score":61.1,"version":"Seed OSS"},{"company":"openai","family":"GPT","model":"gpt-5","place":7,"score":61.09,"version":"GPT-5"},{"company":"antgroup","family":"Ring","model":"ring-flash-2.0","place":8,"score":59.9,"version":"Ring 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":9,"score":59.8,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ring","model":"ring-flash-linear-2.0","place":10,"score":56.6,"version":"Ring 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking-2504","place":11,"score":56.2,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":12,"score":54.1,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":13,"score":54,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":14,"score":53,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":15,"score":52.3,"version":"LLaDA 2.1"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":16,"score":50.8,"version":"GPT-OSS"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":17,"score":50.3,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":18,"score":48.8,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":19,"score":43.6,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":20,"score":41.7,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":21,"score":41.7,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":22,"score":33.2,"version":"Qwen3"}],"stars":"","top_company":"antgroup","top_model":"Ling 1T","top_model_family":"Ling","top_model_id":"ling-1t","top_model_version":"Ling 1T","top_open_source":1,"top_percent":74.46,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":20,"description":"Challenging math benchmark (Omni-MATH-HARD).","github":"","has_recent":0,"id":"omni-math-hard","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Omni-MATH-HARD","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":73.61,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":69.36,"version":"Gemini 2.5"},{"company":"mbzuai","family":"K2","model":"k2-think","place":3,"score":60.73,"version":"K2-THINK"},{"company":"openai","family":"o3","model":"o3-high","place":4,"score":59.39,"version":"o3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":5,"score":58.88,"version":"Nemotron"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":6,"score":57.76,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":56.91,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":8,"score":53.22,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":9,"score":51.33,"version":"DeepSeek R1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":10,"score":46.93,"version":"QwQ"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":11,"score":41.51,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":12,"score":23.87,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":13,"score":13.95,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-e-1b-base","place":14,"score":10.97,"version":"Falcon E"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":15,"score":9.14,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-0.6b","place":16,"score":5.89,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":17,"score":3.93,"version":"Qwen2.5"},{"company":"tii","family":"Falcon","model":"falcon-3-1b-base","place":18,"score":3.32,"version":"Falcon 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-base","place":19,"score":2.64,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-360m-base","place":20,"score":1.21,"version":"SmolLM2"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":73.61,"topic":"Math","total_models":1613,"url":""},{"bench_models":54,"description":"Document understanding benchmark covering multi-page layouts, tables, and charts for robust question answering.","github":"","has_recent":0,"id":"omnidocbench","leaderboard":"","lower_is_better":true,"metric":"percent","name":"OmniDocBench","paper":"","placements":[{"company":"deepseek","family":"DeepSeek OCR","model":"gundam-m","place":1,"score":12.3,"version":"DeepSeek OCR"},{"company":"rednote","family":"dots","model":"dots.ocr","place":2,"score":12.5,"version":"dots"},{"company":"deepseek","family":"DeepSeek OCR","model":"gundam","place":3,"score":12.7,"version":"DeepSeek OCR"},{"company":"opendatalab","family":"MinerU","model":"mineru2.0","place":4,"score":13.3,"version":"MinerU"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-base","place":5,"score":13.7,"version":"DeepSeek OCR"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-large","place":6,"score":13.8,"version":"DeepSeek OCR"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":14.8,"version":"Gemini 2.5"},{"company":"paddlepaddle","family":"PPStructure","model":"ppstructure-v3","place":8,"score":15.2,"version":"PPStructure"},{"company":"kingsoft","family":"MonkeyOCR","model":"monkeyocr-1.2b","place":9,"score":15.4,"version":"MonkeyOCR"},{"company":"opendatalab","family":"MinerU","model":"mineru-2.1.1","place":10,"score":16.2,"version":"MinerU"},{"company":"rednote","family":"dots","model":"dots.ocr","place":11,"score":18.2,"version":"dots"},{"company":"mathpix","family":"Mathpix","model":"mathpix","place":12,"score":19.1,"version":"Mathpix"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":13,"score":21.4,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3-78b","place":14,"score":21.8,"version":"InternVL 3"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-small","place":15,"score":22.1,"version":"DeepSeek OCR"},{"company":"openai","family":"GPT","model":"gpt-4o","place":16,"score":23.3,"version":"GPT-4o"},{"company":"chatdoc","family":"OCRFlux","model":"ocrflux-3b","place":17,"score":23.8,"version":"OCRFlux"},{"company":"stepfun","family":"GOT-OCR","model":"got-ocr2.0","place":18,"score":28.7,"version":"GOT-OCR"},{"company":"datalab","family":"Marker","model":"marker","place":19,"score":29.6,"version":"Marker"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":20,"score":31.6,"version":"Qwen2.5 VL"},{"company":"allenai","family":"OLMOCR","model":"olmocr","place":21,"score":32.6,"version":"OLMOCR"},{"company":"bytedance","family":"Dolphin","model":"dolphin","place":22,"score":35.6,"version":"Dolphin"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-tiny","place":23,"score":38.6,"version":"DeepSeek OCR"},{"company":"opengvlab","family":"Intern","model":"internvl2-76b","place":24,"score":44,"version":"InternVL 2"},{"company":"meta","family":"Nougat","model":"nougat","place":25,"score":45.2,"version":"Nougat"},{"company":"IBM","family":"Docling","model":"smoldocling","place":26,"score":49.3,"version":"SMoLDocling"},{"company":"datalab","family":"Marker","model":"marker-1.8.2","place":27,"score":71.3,"version":"Marker 1.8"},{"company":"opendatalab","family":"MinerU","model":"mineru2-pp","place":28,"score":71.51,"version":"MinerU2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":29,"score":73.1,"version":"Qwen3 VL"},{"company":"bytedance","family":"Dolphin","model":"dolphin","place":30,"score":74.67,"version":"Dolphin"},{"company":"ocrflux","family":"OCRFlux","model":"ocrflux","place":31,"score":74.82,"version":"OCRFlux"},{"company":"openai","family":"GPT","model":"gpt-4o","place":32,"score":75.02,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":33,"score":76.9,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl3","place":34,"score":80.33,"version":"InternVL 3"},{"company":"whu","family":"POINTS","model":"points-reader","place":35,"score":80.98,"version":"POINTS"},{"company":"allenai","family":"OLMOCR","model":"olmocr","place":36,"score":81.79,"version":"OLMOCR"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":37,"score":82,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-241b","place":38,"score":82.67,"version":"InternVL 3.5"},{"company":"bytedance","family":"Dolphin","model":"dolphin-1.5","place":39,"score":83.21,"version":"Dolphin"},{"company":"opendatalab","family":"MinerU","model":"mineru2-vlm","place":40,"score":85.56,"version":"MinerU2"},{"company":"nanonets","family":"Nanonets OCR","model":"nanonets-ocr-s","place":41,"score":85.59,"version":"Nanonets OCR"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":42,"score":85.7,"version":"GPT-5.2"},{"company":"paddlepaddle","family":"PP-Structure","model":"pp-structurev3","place":43,"score":86.73,"version":"PP-Structure V3"},{"company":"monkeyocr","family":"MonkeyOCR","model":"monkeyocr-pro-1.2b","place":44,"score":86.96,"version":"MonkeyOCR"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":45,"score":87.02,"version":"Qwen2.5 VL"},{"company":"monkeyocr","family":"MonkeyOCR","model":"monkeyocr-3b","place":46,"score":87.13,"version":"MonkeyOCR"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr","place":47,"score":87.36,"version":"DeepSeek OCR"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":48,"score":87.5,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":49,"score":87.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":50,"score":88,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":51,"score":88.03,"version":"Gemini 2.5"},{"company":"dots","family":"dots.ocr","model":"dots-ocr","place":52,"score":88.41,"version":"dots.ocr"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":53,"score":88.5,"version":"Gemini 3"},{"company":"ocrverse","family":"OCRVerse","model":"ocrverse","place":54,"score":88.56,"version":"OCRVerse"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":55,"score":88.8,"version":"Kimi K2.5"},{"company":"kingsoft","family":"MonkeyOCR","model":"monkeyocr-pro-3b","place":56,"score":88.85,"version":"MonkeyOCR Pro"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":57,"score":89.15,"version":"Qwen3 VL"},{"company":"opendatalab","family":"MinerU","model":"mineru2.5","place":58,"score":90.67,"version":"MinerU2.5"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-2","place":59,"score":91.09,"version":"DeepSeek OCR 2"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":60,"score":92.6,"version":"Ovis 2.6"},{"company":"paddlepaddle","family":"PaddleOCR","model":"paddleocr-vl","place":61,"score":92.86,"version":"PaddleOCR"}],"stars":"","top_company":"deepseek","top_model":"Gundam-M","top_model_family":"DeepSeek OCR","top_model_id":"gundam-m","top_model_version":"DeepSeek OCR","top_open_source":1,"top_percent":12.3,"topic":"Document understanding","total_models":1613,"url":""},{"bench_models":24,"description":"Document understanding benchmark v1.5 with OCR evaluation. Overall Edit Distance metric, lower is better.","github":"","has_recent":0,"id":"omnidocbench-1.5","leaderboard":"","lower_is_better":true,"metric":"edit_distance","name":"OmniDocBench 1.5","paper":"","placements":[{"company":"bytedance","family":"Dolphin","model":"dolphin-v2","place":1,"score":0.054,"version":"Dolphin V2"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-2","place":2,"score":0.1,"version":"DeepSeek OCR 2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":0.106,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":0.115,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":5,"score":0.121,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr","place":6,"score":0.129,"version":"DeepSeek OCR"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":7,"score":0.143,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":0.145,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":9,"score":0.145,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":10,"score":0.145,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":11,"score":0.145,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":12,"score":0.147,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":0.147,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":14,"score":0.152,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":15,"score":0.154,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":16,"score":77,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":17,"score":84.5,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":18,"score":85.7,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":19,"score":85.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":20,"score":87.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":21,"score":88.5,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":22,"score":88.8,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":23,"score":88.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":24,"score":89.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":25,"score":89.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":26,"score":90.8,"version":"Qwen3.5"}],"stars":"","top_company":"bytedance","top_model":"Dolphin V2","top_model_family":"Dolphin","top_model_id":"dolphin-v2","top_model_version":"Dolphin V2","top_open_source":1,"top_percent":0.054,"topic":"OCR","total_models":1613,"url":""},{"bench_models":25,"description":"Chinese subset of OmniDocBench focusing on OCR-grounded document comprehension and reasoning.","github":"","has_recent":0,"id":"omnidocbench-cn","leaderboard":"","lower_is_better":true,"metric":"percent","name":"OmniDocBench-CN","paper":"","placements":[{"company":"paddlepaddle","family":"PPStructure","model":"ppstructure-v3","place":1,"score":13.6,"version":"PPStructure"},{"company":"deepseek","family":"DeepSeek OCR","model":"gundam-m","place":2,"score":15.7,"version":"DeepSeek OCR"},{"company":"rednote","family":"dots","model":"dots.ocr","place":3,"score":16,"version":"dots"},{"company":"deepseek","family":"DeepSeek OCR","model":"gundam","place":4,"score":18.1,"version":"DeepSeek OCR"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-large","place":5,"score":20.8,"version":"DeepSeek OCR"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":21.2,"version":"Gemini 2.5"},{"company":"opendatalab","family":"MinerU","model":"mineru2.0","place":7,"score":23.8,"version":"MinerU"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-base","place":8,"score":24,"version":"DeepSeek OCR"},{"company":"opendatalab","family":"MinerU","model":"mineru-2.1.1","place":9,"score":24.4,"version":"MinerU"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":10,"score":26.1,"version":"Qwen2.5 VL"},{"company":"rednote","family":"dots","model":"dots.ocr","place":11,"score":26.1,"version":"dots"},{"company":"kingsoft","family":"MonkeyOCR","model":"monkeyocr-1.2b","place":12,"score":26.3,"version":"MonkeyOCR"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-small","place":13,"score":28.4,"version":"DeepSeek OCR"},{"company":"opengvlab","family":"Intern","model":"internvl3-78b","place":14,"score":29.6,"version":"InternVL 3"},{"company":"chatdoc","family":"OCRFlux","model":"ocrflux-3b","place":15,"score":34.9,"version":"OCRFlux"},{"company":"deepseek","family":"DeepSeek OCR","model":"deepseek-ocr-tiny","place":16,"score":36.1,"version":"DeepSeek OCR"},{"company":"mathpix","family":"Mathpix","model":"mathpix","place":17,"score":36.4,"version":"Mathpix"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":18,"score":39.9,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":19,"score":39.9,"version":"GPT-4o"},{"company":"stepfun","family":"GOT-OCR","model":"got-ocr2.0","place":20,"score":41.1,"version":"GOT-OCR"},{"company":"bytedance","family":"Dolphin","model":"dolphin","place":21,"score":44,"version":"Dolphin"},{"company":"opengvlab","family":"Intern","model":"internvl2-76b","place":22,"score":44.3,"version":"InternVL 2"},{"company":"allenai","family":"OLMOCR","model":"olmocr","place":23,"score":46.9,"version":"OLMOCR"},{"company":"datalab","family":"Marker","model":"marker","place":24,"score":49.7,"version":"Marker"},{"company":"IBM","family":"Docling","model":"smoldocling","place":25,"score":81.6,"version":"SMoLDocling"},{"company":"meta","family":"Nougat","model":"nougat","place":26,"score":97.3,"version":"Nougat"}],"stars":"","top_company":"paddlepaddle","top_model":"PPStructure v3","top_model_family":"PPStructure","top_model_id":"ppstructure-v3","top_model_version":"PPStructure","top_open_source":1,"top_percent":13.6,"topic":"Document understanding (Chinese)","total_models":1613,"url":""},{"bench_models":2,"description":"OmniMMI benchmark for multimodal interaction across video streams.","github":"","has_recent":0,"id":"omnimmi","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OmniMMI","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":53,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":2,"score":49.5,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":53,"topic":"Multimodal interaction","total_models":1613,"url":""},{"bench_models":9,"description":"Spatial understanding and reasoning benchmark (OmniSpatial).","github":"","has_recent":0,"id":"omnispatial","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OmniSpatial","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.6v","place":1,"score":52,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":2,"score":51,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":3,"score":50.6,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":4,"score":47.7,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":5,"score":47,"version":"Step 3"},{"company":"google","family":"Gemma","model":"gemma-3","place":6,"score":40.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":7,"score":40.8,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl","place":8,"score":37.3,"version":"Kimi VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":9,"score":37.3,"version":"Kimi VL"}],"stars":"","top_company":"zai","top_model":"GLM-4.6V","top_model_family":"GLM","top_model_id":"glm-4.6v","top_model_version":"GLM 4.6V","top_open_source":1,"top_percent":52,"topic":"Spatial reasoning","total_models":1613,"url":""},{"bench_models":14,"description":"OneIG-Bench English subset score for text-to-image generation.","github":"","has_recent":0,"id":"oneig-bench-en","leaderboard":"","lower_is_better":false,"metric":"score","name":"OneIG-Bench EN","paper":"","placements":[{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":1,"score":0.578,"version":"Nano Banana"},{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":2,"score":0.576,"version":"Seedream"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":3,"score":0.576,"version":"Seedream"},{"company":"zai","family":"Z-Image","model":"z-image","place":4,"score":0.546,"version":"Z-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":5,"score":0.539,"version":"Qwen-Image"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":6,"score":0.533,"version":"GPT Image 1"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":7,"score":0.53,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":8,"score":0.528,"version":"Z-Image"},{"company":"zai","family":"GLM","model":"glm-image","place":9,"score":0.528,"version":"GLM-Image"},{"company":"black-forest-labs","family":"FLUX.1","model":"flux-1-dev","place":10,"score":0.434,"version":"FLUX.1"},{"company":"jondurbin","family":"Bagel","model":"bagel","place":11,"score":0.361,"version":"Bagel"},{"company":"stabilityai","family":"Stable Diffusion","model":"sd-xl","place":12,"score":0.316,"version":"SD XL"},{"company":"unknown","family":"Show-o","model":"show-o2","place":13,"score":0.308,"version":"Show-o2"},{"company":"bytedance","family":"Janus","model":"janus-pro","place":14,"score":0.267,"version":"Janus Pro"}],"stars":"","top_company":"unknown","top_model":"Nano Banana 2.0","top_model_family":"Nano Banana","top_model_id":"nano-banana-2.0","top_model_version":"Nano Banana","top_open_source":0,"top_percent":0.578,"topic":"Text-to-image","total_models":1613,"url":""},{"bench_models":11,"description":"OneIG-Bench Chinese subset score for text-to-image generation.","github":"","has_recent":0,"id":"oneig-bench-zh","leaderboard":"","lower_is_better":false,"metric":"score","name":"OneIG-Bench ZH","paper":"","placements":[{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":1,"score":0.567,"version":"Nano Banana"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":2,"score":0.553,"version":"Seedream"},{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":3,"score":0.551,"version":"Seedream"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":4,"score":0.548,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image","place":5,"score":0.535,"version":"Z-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":6,"score":0.515,"version":"Qwen-Image"},{"company":"zai","family":"GLM","model":"glm-image","place":7,"score":0.511,"version":"GLM-Image"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":8,"score":0.507,"version":"Z-Image"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":9,"score":0.474,"version":"GPT Image 1"},{"company":"jondurbin","family":"Bagel","model":"bagel","place":10,"score":0.37,"version":"Bagel"},{"company":"bytedance","family":"Janus","model":"janus-pro","place":11,"score":0.24,"version":"Janus Pro"}],"stars":"","top_company":"unknown","top_model":"Nano Banana 2.0","top_model_family":"Nano Banana","top_model_id":"nano-banana-2.0","top_model_version":"Nano Banana","top_open_source":0,"top_percent":0.567,"topic":"Text-to-image","total_models":1613,"url":""},{"bench_models":4,"description":"Online web automation and task execution benchmark.","github":"","has_recent":0,"id":"online-mind2web","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Online-Mind2web","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":85.9,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":2,"score":76.4,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":69,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"gpt-03-cua","place":4,"score":61.3,"version":"o3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":85.9,"topic":"Web automation","total_models":1613,"url":""},{"bench_models":2,"description":"Rewrite benchmark assessing open-ended editing and directive-following quality.","github":"","has_recent":0,"id":"open-rewrite","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Open Rewrite","paper":"","placements":[{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":1,"score":51,"version":"MobileLLM"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":2,"score":41.6,"version":"Llama 3.2"}],"stars":"","top_company":"meta","top_model":"MobileLLM P1","top_model_family":"MobileLLM","top_model_id":"mobilellm-p1","top_model_version":"MobileLLM","top_open_source":1,"top_percent":51,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":30,"description":"Open-book multiple choice science questions with supporting facts.","github":"","has_recent":0,"id":"openbookqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OpenBookQA","paper":"","placements":[{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b-pyche","place":1,"score":96.6,"version":"Hermes 4.3"},{"company":"qwen","family":"Qwen","model":"qwen-3","place":2,"score":96.4,"version":"Qwen3"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-70b","place":3,"score":94.8,"version":"Hermes 4.3"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b","place":4,"score":91.8,"version":"Hermes 4.3"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":5,"score":60.2,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":6,"score":58.6,"version":"Llama 1"},{"company":"openai","family":"GPT","model":"gpt-3","place":7,"score":57.6,"version":"GPT-3"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":8,"score":57.2,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":9,"score":56.4,"version":"Llama 1"},{"company":"google","family":"PaLM","model":"palm-540b","place":10,"score":53.4,"version":"PaLM"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":11,"score":52.8,"version":"Gemma 2"},{"company":"google","family":"PaLM","model":"palm-62b","place":12,"score":50.4,"version":"PaLM"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":13,"score":47.8,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":14,"score":47.2,"version":"OLMo 2"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":15,"score":46.6,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":16,"score":46.4,"version":"Marin 32B"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":17,"score":46.2,"version":"Nemotron 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":18,"score":45.8,"version":"Nemotron Nano v2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":19,"score":45,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":20,"score":44.4,"version":"Qwen2.5"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":21,"score":43.6,"version":"MobileLLM P1"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":22,"score":42.4,"version":"SmolLM1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":23,"score":42.2,"version":"SmolLM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":24,"score":40,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":25,"score":38.4,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":26,"score":37.2,"version":"Llama 3.2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":27,"score":34.6,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":28,"score":34.6,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":29,"score":0.556,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":30,"score":0.526,"version":"Llama TFree HAT"}],"stars":"128","top_company":"nousresearch","top_model":"Hermes 4.3 36B Pyche","top_model_family":"Hermes","top_model_id":"hermes-4.3-36b-pyche","top_model_version":"Hermes 4.3","top_open_source":1,"top_percent":96.6,"topic":"Science QA","total_models":1613,"url":"https://github.com/allenai/OpenBookQA"},{"bench_models":3,"description":"OpenRewrite evaluation; micro-averaged RougeL.","github":"","has_recent":0,"id":"openrewrite-eval","leaderboard":"","lower_is_better":false,"metric":"rougel","name":"OpenRewrite-Eval","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":1,"score":46.9,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":2,"score":44.9,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-1b","place":3,"score":39.2,"version":"Llama 1"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5 1.5B Instruct","top_model_family":"Qwen","top_model_id":"qwen2.5-1.5b-instruct","top_model_version":"Qwen2.5","top_open_source":1,"top_percent":46.9,"topic":"Rewrite quality","total_models":1613,"url":""},{"bench_models":5,"description":"OptMATH benchmark targeting challenging math optimization and problem-solving tasks.","github":"","has_recent":0,"id":"optmath","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OptMATH","paper":"","placements":[{"company":"antgroup","family":"Ling","model":"ling-1t","place":1,"score":57.68,"version":"Ling 1T"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":42.77,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":39.16,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":4,"score":35.99,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":5,"score":35.84,"version":"Kimi K2"}],"stars":"","top_company":"antgroup","top_model":"Ling 1T","top_model_family":"Ling","top_model_id":"ling-1t","top_model_version":"Ling 1T","top_open_source":1,"top_percent":57.68,"topic":"Math optimization reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Ordering benchmark requiring models to sequence 15 items correctly.","github":"","has_recent":0,"id":"order-15-items","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Order 15 Items","paper":"","placements":[{"company":"mbzuai","family":"K2","model":"k2-v2","place":1,"score":87.6,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":2,"score":37,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":3,"score":25,"version":"OLMo 3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":4,"score":4.7,"version":"K2 V2"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":5,"score":4.5,"version":"Llama 3.1"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":6,"score":3.5,"version":"Llama 3.0"}],"stars":"","top_company":"mbzuai","top_model":"K2-V2","top_model_family":"K2","top_model_id":"k2-v2","top_model_version":"K2 V2","top_open_source":1,"top_percent":87.6,"topic":"List ordering","total_models":1613,"url":""},{"bench_models":5,"description":"Ordering benchmark requiring models to sequence 30 items correctly.","github":"","has_recent":0,"id":"order-30-items","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Order 30 Items","paper":"","placements":[{"company":"mbzuai","family":"K2","model":"k2-v2","place":1,"score":40.3,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":2,"score":0.7,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":3,"score":0.6,"version":"OLMo 3"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":4,"score":0.2,"version":"Llama 3.0"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":5,"score":0.1,"version":"Llama 3.1"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":6,"score":0,"version":"K2 V2"}],"stars":"","top_company":"mbzuai","top_model":"K2-V2","top_model_family":"K2","top_model_id":"k2-v2","top_model_version":"K2 V2","top_open_source":1,"top_percent":40.3,"topic":"List ordering (long)","total_models":1613,"url":""},{"bench_models":23,"description":"Agentic GUI task completion and grounding on desktop environments.","github":"","has_recent":0,"id":"osworld","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OSWorld","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":72.7,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":72.7,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":66.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":66.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":62.9,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":6,"score":61.9,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":7,"score":61.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":61.4,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":9,"score":61,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":10,"score":50.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":11,"score":44.4,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":12,"score":42.2,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":13,"score":42.2,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":14,"score":38.1,"version":"Qwen3 VL"},{"company":"openai","family":"o3","model":"gpt-03-cua","place":15,"score":38.1,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":16,"score":37.2,"version":"GLM 4.6V"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":17,"score":36.7,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":18,"score":36.7,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":19,"score":35,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":20,"score":34.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":21,"score":33.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":22,"score":32.6,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":23,"score":30.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":24,"score":26.2,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":25,"score":21.1,"version":"GLM 4.6V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":26,"score":13.3,"version":"Gemini 2.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":72.7,"topic":"GUI agents","total_models":1613,"url":""},{"bench_models":29,"description":"OSWorld-G center accuracy (no_refusal).","github":"","has_recent":0,"id":"osworld-g","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OSWorld-G","paper":"","placements":[{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":1,"score":71.8,"version":"Holo1.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":2,"score":70.4,"version":"UI-Venus"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":3,"score":68.3,"version":"Qwen3 VL"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":4,"score":66.27,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":5,"score":65.1,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":62.9,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":7,"score":61.96,"version":"Qwen2.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":8,"score":61.57,"version":"Holo1.5"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":9,"score":61.4,"version":"UI-TARS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":10,"score":59.6,"version":"Claude 4"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":11,"score":59.02,"version":"Step3-VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":12,"score":58.8,"version":"UI-Venus"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":13,"score":58.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":14,"score":58.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":15,"score":56.7,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":16,"score":56.1,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":17,"score":54.71,"version":"GLM 4.6V"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":18,"score":54.7,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":19,"score":50.54,"version":"MiMo VL"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":20,"score":47.25,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":21,"score":46.1,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":22,"score":45.2,"version":"Gemini 2.5"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":23,"score":42.16,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":24,"score":40.59,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":25,"score":37.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":26,"score":34.31,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":27,"score":31.91,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":28,"score":30.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":29,"score":8.8,"version":"Qwen2.5 VL"}],"stars":"","top_company":"hcompany","top_model":"Holo1.5-72B","top_model_family":"Holo","top_model_id":"holo1.5-72b","top_model_version":"Holo1.5","top_open_source":1,"top_percent":71.8,"topic":"GUI agents","total_models":1613,"url":""},{"bench_models":13,"description":"Verified subset of OSWorld GUI agent benchmark.","github":"","has_recent":0,"id":"osworld-verified","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OSWorld Verified","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":72.7,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":2,"score":72.5,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":66.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":4,"score":64.7,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":5,"score":64.7,"version":"GPT-5.3 Codex"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":63.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":7,"score":62.2,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":61.4,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":9,"score":58,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":10,"score":56.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":11,"score":54.5,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":12,"score":38.2,"version":"GPT-5.2 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":13,"score":38.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":14,"score":38.2,"version":"GPT-5.2 Codex"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":15,"score":38.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":16,"score":37.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":17,"score":37.9,"version":"GPT-5.2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":72.7,"topic":"GUI agents","total_models":1613,"url":""},{"bench_models":4,"description":"Second-generation OSWorld GUI agent benchmark.","github":"","has_recent":0,"id":"osworld2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OSWorld2","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.5v","place":1,"score":35.8,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":2,"score":14.9,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":3,"score":8.2,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":4,"score":6.2,"version":"Gemma 3"}],"stars":"","top_company":"zai","top_model":"GLM-4.5V","top_model_family":"GLM","top_model_id":"glm-4.5v","top_model_version":"GLM 4.5V","top_open_source":1,"top_percent":35.8,"topic":"GUI agents","total_models":1613,"url":""},{"bench_models":4,"description":"Open-vocabulary benchmark for streaming video understanding.","github":"","has_recent":0,"id":"ovbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OVBench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":65.1,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":62.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":61.2,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":60,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":65.1,"topic":"Open-vocabulary streaming","total_models":1613,"url":""},{"bench_models":4,"description":"Streaming video QA benchmark with open-vocabulary queries.","github":"","has_recent":0,"id":"ovobench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"OVOBench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":72.6,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":2,"score":72.3,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":70.9,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":70.1,"version":"Gemini 3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":72.6,"topic":"Streaming video QA","total_models":1613,"url":""},{"bench_models":6,"description":"PaperBench developer subset measuring code reasoning accuracy.","github":"","has_recent":0,"id":"paper-bench-code-dev","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PaperBench Code-Dev","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":1,"score":43.3,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":2,"score":29.9,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":3,"score":27.8,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":4,"score":13.2,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":5,"score":12.2,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":6,"score":5.7,"version":"Gemini 2.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4","top_model_family":"Claude","top_model_id":"claude-sonnet-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":43.3,"topic":"Code understanding","total_models":1613,"url":""},{"bench_models":4,"description":"Benchmark for understanding and reasoning over research papers.","github":"","has_recent":0,"id":"paperbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PaperBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":1,"score":72.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":2,"score":63.7,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":3,"score":63.5,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":47.1,"version":"DeepSeek V3.2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5 Thinking","top_model_family":"Claude","top_model_id":"claude-opus-4.5-thinking","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":72.9,"topic":"Research paper understanding","total_models":1613,"url":""},{"bench_models":22,"description":"Physics reasoning and calculation benchmark.","github":"","has_recent":0,"id":"phybench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PHYBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":80,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":77,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":3,"score":74,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":4,"score":74,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":5,"score":73,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":69,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":7,"score":60,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":59,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":56,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":48,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":48,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":12,"score":41,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":13,"score":40,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":31,"version":"Claude 4.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":15,"score":30.06,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":16,"score":29.84,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":17,"score":28.23,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":18,"score":27.67,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":19,"score":26.04,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":20,"score":14.59,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":21,"score":13.05,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":22,"score":12.75,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":23,"score":11.7,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":24,"score":9.76,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":80,"topic":"Physics reasoning","total_models":1613,"url":""},{"bench_models":5,"description":"Multimodal physics reasoning benchmark (PhyX).","github":"","has_recent":0,"id":"phyx","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PhyX","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":1,"score":59.45,"version":"Step3-VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":2,"score":57.67,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":3,"score":56,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":4,"score":52.28,"version":"GLM 4.6V"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":5,"score":50.51,"version":"InternVL 3.5"}],"stars":"","top_company":"stepfun","top_model":"Step3-VL-10B","top_model_family":"Step","top_model_id":"step3-vl-10b","top_model_version":"Step3-VL","top_open_source":1,"top_percent":59.45,"topic":"Physics reasoning (multimodal)","total_models":1613,"url":""},{"bench_models":70,"description":"Physical commonsense about everyday tasks and object affordances.","github":"","has_recent":0,"id":"piqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PIQA","paper":"","placements":[{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":1,"score":96.5,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":2,"score":92.44,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":3,"score":92.17,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":4,"score":91.95,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":5,"score":91.57,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":6,"score":88.3,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":7,"score":87.54,"version":"Ling 2.0"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":8,"score":87.1,"version":"GLM 4.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":9,"score":86.89,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":10,"score":86.51,"version":"LLaDA 2.0"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":11,"score":86.1,"version":"Marin 32B"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":12,"score":85.9,"version":"Marin 32B"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":13,"score":85.6,"version":"OLMo 3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":14,"score":84.87,"version":"LLaDA 2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":15,"score":84.7,"version":"DeepSeek V3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":16,"score":84.6,"version":"K2 V2"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":17,"score":84.6,"version":"Llama 3.0"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":18,"score":84.33,"version":"Nemotron 3"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":19,"score":84.3,"version":"Llama 3.1"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":20,"score":84.2,"version":"K2 V2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":21,"score":84.1,"version":"Gemma 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":22,"score":83.6,"version":"Mixtral"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":23,"score":83.5,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":24,"score":83.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":25,"score":83.3,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":26,"score":83.1,"version":"OLMo 2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":27,"score":83,"version":"Mistral"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":28,"score":82.8,"version":"Nemotron Nano v2"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":29,"score":82.8,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":30,"score":82.6,"version":"Llama 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":31,"score":82.4,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":32,"score":82.3,"version":"Llama 1"},{"company":"google","family":"PaLM","model":"palm-540b","place":33,"score":82.3,"version":"PaLM"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":34,"score":82.2,"version":"Mistral"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":35,"score":81.8,"version":"Gemma 3"},{"company":"google","family":"Gopher","model":"gopher-280b","place":36,"score":81.8,"version":"Gopher"},{"company":"google","family":"Chinchilla","model":"chinchilla-70b","place":37,"score":81.8,"version":"Chinchilla"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":38,"score":81.6,"version":"Motif 2"},{"company":"google","family":"PaLM","model":"palm-cont-62b","place":39,"score":81.4,"version":"PaLM"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":40,"score":81.2,"version":"Gemma 2"},{"company":"openai","family":"GPT","model":"gpt-3","place":41,"score":81,"version":"GPT-3"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":42,"score":80.8,"version":"Llama 2"},{"company":"google","family":"PaLM","model":"palm-62b","place":43,"score":80.5,"version":"PaLM"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":44,"score":80.1,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":45,"score":79.8,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":46,"score":77.9,"version":"Llama 2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":47,"score":77.6,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":48,"score":77.3,"version":"Gemma 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":49,"score":76.55,"version":"MobileLLM P1"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":50,"score":76.3,"version":"Pythia"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":51,"score":76.1,"version":"Qwen2.5"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":52,"score":76,"version":"SmolLM1"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":53,"score":76,"version":"Pythia"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":54,"score":75.14,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":55,"score":74.8,"version":"Llama 1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-instruct","place":56,"score":74.4,"version":"SmolLM2"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":57,"score":74.1,"version":"Pythia"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":58,"score":73.8,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-instruct","place":59,"score":73.2,"version":"Qwen2.5"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":60,"score":72.8,"version":"CodeLlama-7B"},{"company":"meta","family":"Llama","model":"llama-1b","place":61,"score":72.3,"version":"Llama 1"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":62,"score":72,"version":"Pythia"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-instruct","place":63,"score":71.6,"version":"SmolLM1"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":64,"score":70,"version":"Pythia"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":65,"score":68.4,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":66,"score":68.4,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":67,"score":67.7,"version":"Gemma 3"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":68,"score":67.5,"version":"Pythia"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-instruct","place":69,"score":66.3,"version":"SmolLM2"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":70,"score":66.2,"version":"Gemma 3"},{"company":"loka","family":"TRLM","model":"trlm-135m","place":71,"score":64.91,"version":"TRLM"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":72,"score":61.8,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":73,"score":59.8,"version":"Pythia"}],"stars":"","top_company":"antgroup","top_model":"LLaDA2.0 Flash","top_model_family":"LLaDA","top_model_id":"llada2.0-flash","top_model_version":"LLaDA 2.0","top_open_source":1,"top_percent":96.5,"topic":"Physical commonsense","total_models":1613,"url":"https://github.com/ybisk/piqa"},{"bench_models":53,"description":"Counting objects/instances in images (PixmoCount).","github":"","has_recent":0,"id":"pixmocount","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PixmoCount","paper":"https://arxiv.org/abs/2409.17146v2","placements":[{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":1,"score":90.2,"version":"Eagle 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":90,"version":"Gemini 3"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":3,"score":88.9,"version":"Molmo2"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":4,"score":88.5,"version":"Molmo2"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":5,"score":88.1,"version":"Molmo2"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":6,"score":85.2,"version":"Molmo"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":7,"score":84.8,"version":"Molmo"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":8,"score":83.3,"version":"Molmo"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":9,"score":79.6,"version":"MolmoE"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":10,"score":79.4,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":11,"score":79.4,"version":"MiMo VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":12,"score":74.4,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":73.8,"version":"Gemini 2.5"},{"company":"unknown","family":"PLM","model":"plm-8b","place":14,"score":68,"version":"PLM"},{"company":"openai","family":"GPT","model":"gpt-5","place":15,"score":67.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":16,"score":65,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":17,"score":64.3,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":18,"score":63.9,"version":"Gemini 2.5"},{"company":"unknown","family":"PLM","model":"plm-3b","place":19,"score":63,"version":"PLM"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":20,"score":62.8,"version":"MiniCPM V 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":21,"score":62.4,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":22,"score":62,"version":"InternVL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":23,"score":61.9,"version":"InternVL 3.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":24,"score":61.1,"version":"Gemini 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":25,"score":60.7,"version":"LLaVA OneVision"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":26,"score":60.7,"version":"Qwen2.5 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":27,"score":60.7,"version":"GLM 4.1V"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":28,"score":60,"version":"PaliGemma"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":29,"score":59.6,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":30,"score":58.5,"version":"Llama 3.2"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":31,"score":58.3,"version":"Claude 3.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":32,"score":58.3,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":33,"score":58,"version":"Qwen3 VL"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":34,"score":57.4,"version":"Keye-VL 1.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":35,"score":55.7,"version":"Qwen2 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":36,"score":54.6,"version":"InternVL 2"},{"company":"openai","family":"GPT","model":"gpt-4o","place":37,"score":54.4,"version":"GPT-4o"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":38,"score":54.4,"version":"LLaVA OneVision"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":39,"score":53.5,"version":"Claude 3.7"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":40,"score":51.7,"version":"Pixtral"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":41,"score":50.7,"version":"Cambrian 1"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":42,"score":50.2,"version":"xGen-MM"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":43,"score":48.6,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":44,"score":48,"version":"Qwen2 VL"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":45,"score":47.4,"version":"Llama 3.2"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":46,"score":46.6,"version":"Cambrian 1"},{"company":"openai","family":"GPT","model":"gpt-4v","place":47,"score":45,"version":"GPT-4V"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":48,"score":43.9,"version":"Claude 3"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":49,"score":43.9,"version":"InternVL 2"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":50,"score":43.3,"version":"Claude 3"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":51,"score":38.3,"version":"Phi 3.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":52,"score":35.2,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":53,"score":27.6,"version":"LLaVA 1.5"}],"stars":"","top_company":"unknown","top_model":"Eagle2.5-8B","top_model_family":"Eagle","top_model_id":"eagle2.5-8b","top_model_version":"Eagle 2.5","top_open_source":1,"top_percent":90.2,"topic":"Visual counting","total_models":1613,"url":"https://huggingface.co/datasets/allenai/pixmo-count"},{"bench_models":11,"description":"PubMed Central visual question answering benchmark for biomedical images.","github":"","has_recent":0,"id":"pmc-vqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PMC-VQA","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":64.2,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":2,"score":63.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":63.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":62.4,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":62.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":62,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":59.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":58.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":55.9,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":10,"score":41.2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":11,"score":36.3,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":64.2,"topic":"Medical VQA","total_models":1613,"url":""},{"bench_models":6,"description":"Benchmark for pointing and counting objects in images.","github":"","has_recent":0,"id":"point-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Point-Bench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":85.5,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":76.5,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":74.9,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":62.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":41.8,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":13.6,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":85.5,"topic":"Pointing and counting","total_models":1613,"url":""},{"bench_models":40,"description":"Polyglot mathematics benchmark assessing cross-topic math reasoning.","github":"","has_recent":0,"id":"polymath","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PolyMATH","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":81.6,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":79,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":73.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":71.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":68.9,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":67.3,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":7,"score":65.1,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":8,"score":64.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":9,"score":64.4,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":62.5,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":11,"score":60.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":12,"score":60.1,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":13,"score":59.5,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":14,"score":56.3,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":15,"score":54.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":16,"score":54,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":17,"score":54,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":18,"score":52.8,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":19,"score":52.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":20,"score":51.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":21,"score":51.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":22,"score":50.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":23,"score":49.9,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":24,"score":49.8,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":25,"score":49.8,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":26,"score":48.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":27,"score":47.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":28,"score":47.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":29,"score":45.9,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":30,"score":44.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":31,"score":43.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":32,"score":43.1,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":33,"score":43.1,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":34,"score":39.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":35,"score":37.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":36,"score":31.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":37,"score":30.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":38,"score":28.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":39,"score":27,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":40,"score":25.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":41,"score":22.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":42,"score":18.8,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":81.6,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"Vision-language hallucination benchmark focusing on object existence verification.","github":"https://github.com/AoiDragon/POPE","has_recent":0,"id":"pope","leaderboard":"","lower_is_better":false,"metric":"percent","name":"POPE","paper":"https://arxiv.org/abs/2305.10355","placements":[{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":1,"score":90.1,"version":"InternVL 3"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":2,"score":89.01,"version":"LFM2-VL"},{"company":"moondream","family":"Moondream","model":"moondream-9b-a2b","place":3,"score":89,"version":"Moondream 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":88.4,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":5,"score":88.1,"version":"Gemini 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":6,"score":87.17,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":7,"score":86.17,"version":"Qwen2.5 VL"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":8,"score":85.1,"version":"SmolVLM2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":9,"score":84.6,"version":"Claude 4"}],"stars":"","top_company":"opengvlab","top_model":"InternVL3-2B","top_model_family":"Intern","top_model_id":"internvl3-2b","top_model_version":"InternVL 3","top_open_source":1,"top_percent":90.1,"topic":"Hallucination detection","total_models":1613,"url":""},{"bench_models":41,"description":"Open-domain popular culture question answering benchmark testing long-tail factual recall.","github":"https://github.com/facebookresearch/PopQA","has_recent":0,"id":"popqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PopQA","paper":"https://arxiv.org/abs/2211.07239","placements":[{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-sft","place":1,"score":55.7,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b","place":2,"score":55.5,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-405b-dpo","place":3,"score":55.4,"version":"Tulu 3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-405b","place":4,"score":54.2,"version":"Hermes 3"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-24","place":5,"score":53.6,"version":"GPT-4o"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":6,"score":52.9,"version":"Llama 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":7,"score":48.6,"version":"Tulu 3"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":8,"score":48.1,"version":"Hermes 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":9,"score":46.5,"version":"Tulu 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":10,"score":46.4,"version":"Llama 3.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":11,"score":46.3,"version":"Tulu 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":12,"score":44.9,"version":"DeepSeek V3"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":13,"score":37.2,"version":"OLMo 2"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":14,"score":37,"version":"OLMo 3"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":15,"score":36.4,"version":"Llama 3.1 Nemotron"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":16,"score":33.1,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":17,"score":32.2,"version":"Qwen3 VL"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":18,"score":31.9,"version":"OLMo 3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":19,"score":30.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":20,"score":30.6,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":21,"score":30.4,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":22,"score":30.2,"version":"Gemma 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":23,"score":29.3,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":24,"score":29.3,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":25,"score":29.1,"version":"Tulu 3"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":26,"score":28.76,"version":"Granite 3.1"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":27,"score":28.3,"version":"Gemma 2"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":28,"score":28.04,"version":"Granite 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":29,"score":28,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":30,"score":26.7,"version":"DeepSeek R1"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":31,"score":26.17,"version":"Granite 3.3"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":32,"score":22.93,"version":"Granite 4.0"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":33,"score":22.5,"version":"Magpie"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":34,"score":20.56,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":35,"score":20.55,"version":"Granite 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":36,"score":20.2,"version":"Llama 3.1"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":37,"score":20.2,"version":"Ministral"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":38,"score":18.4,"version":"Granite 3.3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":39,"score":18.1,"version":"Qwen2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":40,"score":13.25,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":41,"score":9.94,"version":"DeepSeek R1"}],"stars":"","top_company":"allenai","top_model":"Llama 3.1 Tulu 3 405B SFT","top_model_family":"Llama","top_model_id":"llama-3.1-tulu-3-405b-sft","top_model_version":"Tulu 3","top_open_source":1,"top_percent":55.7,"topic":"Knowledge / QA","total_models":1613,"url":"https://github.com/facebookresearch/PopQA"},{"bench_models":5,"description":"Measures how well AI agents can post-train base LLMs under fixed compute/time constraints; average score across AIME 2025, BFCL, GPQA Main, GSM8K, and HumanEval.","github":"https://github.com/aisa-group/PostTrainBench","has_recent":0,"id":"posttrainbench","leaderboard":"https://posttrainbench.com/","lower_is_better":false,"metric":"percent","name":"PostTrainBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.1-codex-max","place":1,"score":34.94,"version":"GPT-5.1 Codex"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":20.1,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":18.04,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":17.48,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":14.67,"version":"Claude 4.5"}],"stars":"","top_company":"openai","top_model":"GPT-5.1 Codex-Max","top_model_family":"GPT","top_model_id":"gpt-5.1-codex-max","top_model_version":"GPT-5.1 Codex","top_open_source":0,"top_percent":34.94,"topic":"Post-training automation","total_models":1613,"url":"https://posttrainbench.com/"},{"bench_models":2,"description":"Product Requirements Document benchmark for evaluating agentic coding capabilities.","github":"","has_recent":0,"id":"prdbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PRDBench","paper":"","placements":[{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":1,"score":39.63,"version":"LongCat-Flash-Lite"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":2,"score":15.36,"version":"Qwen3 Next"}],"stars":"","top_company":"meituan","top_model":"LongCat-Flash-Lite","top_model_family":"LongCat","top_model_id":"longcat-flash-lite","top_model_version":"LongCat-Flash-Lite","top_open_source":1,"top_percent":39.63,"topic":"Agentic coding","total_models":1613,"url":""},{"bench_models":9,"description":"Procedural reasoning benchmark evaluating step-by-step logical reasoning.","github":"","has_recent":0,"id":"procbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ProcBench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":1,"score":96.6,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":95,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":92.5,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":4,"score":92.4,"version":"Seed 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":90,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":6,"score":90,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":87.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":8,"score":87.5,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":80.1,"version":"Seed 2.0"}],"stars":"","top_company":"bytedance","top_model":"Seed2.0 Pro","top_model_family":"Seed","top_model_id":"seed2.0-pro","top_model_version":"Seed 2.0","top_open_source":0,"top_percent":96.6,"topic":"Procedural reasoning","total_models":1613,"url":""},{"bench_models":10,"description":"Probing ontological reasoning via question answering.","github":"","has_recent":0,"id":"prontoqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"PrOntoQA","paper":"","placements":[{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":1,"score":97.88,"version":"Ling 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":2,"score":97.12,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":3,"score":97,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":4,"score":96.5,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":5,"score":96.06,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":6,"score":95,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":7,"score":93.12,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":8,"score":87.5,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":9,"score":86,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":10,"score":84.5,"version":"LLaDA 2.1"}],"stars":"","top_company":"antgroup","top_model":"Ling Flash 2.0","top_model_family":"Ling","top_model_id":"ling-flash-2.0","top_model_version":"Ling 2.0","top_open_source":1,"top_percent":97.88,"topic":"Logical reasoning","total_models":1613,"url":""},{"bench_models":9,"description":"Advanced mathematical proof benchmark covering complex theorem proving tasks.","github":"","has_recent":0,"id":"proofbench-advanced","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ProofBench Advanced","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-deep-think-imo-gold","place":1,"score":65.7,"version":"Gemini Deep Think"},{"company":"deepseek","family":"DeepSeekMath","model":"deepseekmath-v2-heavy","place":2,"score":61.9,"version":"DeepSeekMath V2"},{"company":"google","family":"Gemini","model":"gemini-2.5-deep-think","place":3,"score":37.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":20,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4","place":5,"score":18.6,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":17.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":5.2,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":8,"score":4.8,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":9,"score":3.8,"version":"DeepSeek R1"}],"stars":"","top_company":"google","top_model":"Gemini Deep Think (IMO Gold)","top_model_family":"Gemini","top_model_id":"gemini-deep-think-imo-gold","top_model_version":"Gemini Deep Think","top_open_source":0,"top_percent":65.7,"topic":"Mathematical proofs (advanced)","total_models":1613,"url":""},{"bench_models":9,"description":"Entry-level mathematical proof benchmarking set.","github":"","has_recent":0,"id":"proofbench-basic","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ProofBench Basic","paper":"","placements":[{"company":"deepseek","family":"DeepSeekMath","model":"deepseekmath-v2-heavy","place":1,"score":99,"version":"DeepSeekMath V2"},{"company":"google","family":"Gemini","model":"gemini-deep-think-imo-gold","place":2,"score":89,"version":"Gemini Deep Think"},{"company":"google","family":"Gemini","model":"gemini-2.5-deep-think","place":3,"score":83.8,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":59,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":55.2,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4","place":6,"score":46.7,"version":"Grok 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":33.3,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":8,"score":29,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":9,"score":27.1,"version":"Claude 4"}],"stars":"","top_company":"deepseek","top_model":"DeepSeekMath-V2-Heavy","top_model_family":"DeepSeekMath","top_model_id":"deepseekmath-v2-heavy","top_model_version":"DeepSeekMath V2","top_open_source":1,"top_percent":99,"topic":"Mathematical proofs","total_models":1613,"url":""},{"bench_models":2,"description":"Protocol question answering benchmark evaluating understanding of scientific protocols and procedures.","github":"","has_recent":0,"id":"protocolqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ProtocolQA","paper":"","placements":[{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":1,"score":79,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":2,"score":76,"version":"Grok 4"}],"stars":"","top_company":"xai","top_model":"Grok 4.1 (Thinking)","top_model_family":"Grok","top_model_id":"grok-4.1-thinking","top_model_version":"Grok 4.1","top_open_source":0,"top_percent":79,"topic":"Protocol understanding and QA","total_models":1613,"url":""},{"bench_models":5,"description":"Question answering in context.","github":"","has_recent":0,"id":"quac","leaderboard":"","lower_is_better":false,"metric":"percent","name":"QuAC","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":1,"score":53.6,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":2,"score":53.6,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":3,"score":51.1,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":4,"score":51.1,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":5,"score":44.9,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":6,"score":44.4,"version":"Llama 3"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B Base","top_model_family":"Llama","top_model_id":"llama-3.1-405b-base","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":53.6,"topic":"Conversational QA","total_models":1613,"url":"https://quac.ai/"},{"bench_models":2,"description":"Long-document multiple-choice reading comprehension benchmark.","github":"https://github.com/nyu-mll/quality","has_recent":0,"id":"quality","leaderboard":"","lower_is_better":false,"metric":"log_acc","name":"QuALITY","paper":"https://arxiv.org/abs/2112.08608","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":48.8,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":2,"score":45.9,"version":"Llama 3.3"}],"stars":"","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":48.8,"topic":"Long-context reading comprehension","total_models":1613,"url":"https://github.com/nyu-mll/quality"},{"bench_models":5,"description":"English exams for middle and high school.","github":"","has_recent":0,"id":"race","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RACE","paper":"","placements":[{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":1,"score":88.04,"version":"Nemotron 3"},{"company":"radicalnumerics","family":"RND1","model":"rnd1-base-0910","place":2,"score":57.6,"version":"RND1"},{"company":"huawei","family":"Dream","model":"dream-7b-instruct","place":3,"score":44.7,"version":"Dream 7B"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":4,"score":40.2,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada-8b-instruct","place":5,"score":38.7,"version":"LLaDA"}],"stars":"","top_company":"nvidia","top_model":"Nemotron-3-Nano-30B-A3B-Base","top_model_family":"Nemotron","top_model_id":"nemotron-3-nano-30b-a3b-base","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":88.04,"topic":"Reading comprehension","total_models":1613,"url":"http://www.cs.cmu.edu/~glai1/data/race/"},{"bench_models":8,"description":"Randomly constructed complex task environments for agent generalization.","github":"","has_recent":0,"id":"random-complex-tasks","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Random Complex Tasks","paper":"","placements":[{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":1,"score":35.8,"version":"LongCat-Flash"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":2,"score":32.7,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":3,"score":32.6,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":32.5,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":32.5,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":6,"score":29.7,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":7,"score":25.3,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":8,"score":17.2,"version":"GPT-5.2"}],"stars":"","top_company":"meituan","top_model":"LongCat-Flash-Thinking-2601","top_model_family":"LongCat","top_model_id":"longcat-flash-thinking-2601","top_model_version":"LongCat-Flash","top_open_source":1,"top_percent":35.8,"topic":"Agentic tasks (random)","total_models":1613,"url":""},{"bench_models":5,"description":"Real-world browsing and QA benchmark.","github":"","has_recent":0,"id":"realbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Realbench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":49.1,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":2,"score":46,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":39.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":38.4,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"gpt-03-cua","place":5,"score":34.8,"version":"o3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":49.1,"topic":"Web browsing","total_models":1613,"url":""},{"bench_models":99,"description":"Visual question answering with real-world images and scenarios.","github":"https://github.com/ZhuoyuWei/RealWorldQAPlatform","has_recent":0,"id":"realworldqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RealWorldQA","paper":"https://arxiv.org/abs/2410.14332","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":1,"score":85.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":2,"score":84.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":83.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":83.7,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":83.3,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":83.3,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":7,"score":82.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":8,"score":81.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":9,"score":81.3,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":10,"score":81,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":11,"score":80.8,"version":"GPT-5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":12,"score":79.1,"version":"dots.vlm1"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":13,"score":79.08,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":14,"score":79,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":15,"score":79,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":16,"score":78.69,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":17,"score":78.43,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":78,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":19,"score":77.8,"version":"Qwen2 VL"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":20,"score":77.6,"version":"Molmo2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":21,"score":77.4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":22,"score":77,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":23,"score":77,"version":"Claude 4.5"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":24,"score":76.7,"version":"Eagle 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":25,"score":75.9,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":26,"score":75.7,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":27,"score":75.4,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o","place":28,"score":75.4,"version":"GPT-4o"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":29,"score":75.4,"version":"Molmo2"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":30,"score":75.2,"version":"Molmo"},{"company":"unknown","family":"PLM","model":"plm-8b","place":31,"score":75,"version":"PLM"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":32,"score":74.5,"version":"InternVL 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":33,"score":74.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":34,"score":73.7,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":35,"score":73.6,"version":"Gemini 3"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":36,"score":73.6,"version":"Molmo2"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":37,"score":73.5,"version":"Keye-VL 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":38,"score":73.3,"version":"GPT-5"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":39,"score":72.8,"version":"Bagel"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":40,"score":72.7,"version":"InternVL 2"},{"company":"unknown","family":"PLM","model":"plm-3b","place":41,"score":72.4,"version":"PLM"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":42,"score":72.1,"version":"MiniCPM V 4.5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":43,"score":71.9,"version":"LLaVA OneVision"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":44,"score":71.5,"version":"Qwen3 VL"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":45,"score":71.37,"version":"LFM2-VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":46,"score":70.9,"version":"Qwen3 VL"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":47,"score":70.7,"version":"Molmo"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":48,"score":70.7,"version":"GLM 4.1V"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":49,"score":70.6,"version":"InternVL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":50,"score":70.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":51,"score":70.4,"version":"Gemini 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":52,"score":70.3,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":53,"score":70.1,"version":"Qwen2 VL"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":54,"score":70.1,"version":"Manzano"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":55,"score":69.9,"version":"Claude 4.1"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":56,"score":69.8,"version":"Llama 3.2"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":57,"score":69,"version":"MM1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":58,"score":68.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":59,"score":68.1,"version":"Claude 4"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":60,"score":67.8,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":61,"score":67.8,"version":"Cambrian"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":62,"score":67.5,"version":"Gemini 1.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":63,"score":67.5,"version":"Molmo"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":64,"score":67.5,"version":"InternVL 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":65,"score":67.2,"version":"FastVLM"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":66,"score":66.3,"version":"LLaVA OneVision"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":67,"score":66.3,"version":"InternVL 3.5"},{"company":"liquidai","family":"LFM","model":"lfm2-vl-1.6b","place":68,"score":65.75,"version":"LFM2-VL"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":69,"score":65.4,"version":"Pixtral"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":70,"score":65.4,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":71,"score":65.23,"version":"Qwen2.5 VL"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":72,"score":65.1,"version":"Manzano"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":73,"score":65.1,"version":"InternVL 3"},{"company":"liquidai","family":"LFM","model":"lfm2.5-vl-1.6b","place":74,"score":64.84,"version":"LFM2.5-VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":75,"score":64.3,"version":"InternVL 2.5"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":76,"score":64.2,"version":"Cambrian 1"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":77,"score":64.2,"version":"InternVL 2"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":78,"score":64.1,"version":"Llama 3.2"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":79,"score":64.1,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":80,"score":63.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":81,"score":62.6,"version":"Qwen2.5 Omni"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":82,"score":62,"version":"InternVL 3.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":83,"score":61.56,"version":"FastVLM"},{"company":"openai","family":"GPT","model":"gpt-4v","place":84,"score":61.4,"version":"GPT-4V"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":85,"score":61.2,"version":"FastVLM"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":86,"score":61.2,"version":"xGen-MM"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":87,"score":61.1,"version":"Claude 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":88,"score":60.78,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":89,"score":60.7,"version":"GPT-5"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":90,"score":60.5,"version":"BLIP-3"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":91,"score":60.4,"version":"MolmoE"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":92,"score":60.4,"version":"BLIP-3O"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":93,"score":60.1,"version":"Claude 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":94,"score":60.1,"version":"InternVL 2.5"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":95,"score":59.4,"version":"Phi-3"},{"company":"apple","family":"MM1","model":"mm1-30b","place":96,"score":59.4,"version":"MM1"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":97,"score":57.5,"version":"SmolVLM2"},{"company":"baai","family":"Emu","model":"emu3-8b","place":98,"score":57.4,"version":"Emu3"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-1b","place":99,"score":57.12,"version":"InternVL 3.5"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":100,"score":56.9,"version":"MM1.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":101,"score":56.1,"version":"FastVLM"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":102,"score":55.8,"version":"MiniCPM V 2.0"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":103,"score":55.3,"version":"LLaVA 1.5"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":104,"score":55.2,"version":"PaliGemma"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":105,"score":54.8,"version":"LLaVA 1.5"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":106,"score":53.6,"version":"Phi 3.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":107,"score":49.8,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":108,"score":45.5,"version":"Claude 3"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-122B-A10B","top_model_family":"Qwen","top_model_id":"qwen3.5-122b-a10b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":85.1,"topic":"Real-world visual QA","total_models":1613,"url":"https://realworldqa.github.io/"},{"bench_models":2,"description":"Ref-L4 referring expression comprehension on the test split.","github":"","has_recent":0,"id":"ref-l4-test","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ref-L4 (test)","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.6v","place":1,"score":88.9,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":2,"score":87.7,"version":"GLM 4.6V"}],"stars":"","top_company":"zai","top_model":"GLM-4.6V","top_model_family":"GLM","top_model_id":"glm-4.6v","top_model_version":"GLM 4.6V","top_open_source":1,"top_percent":88.9,"topic":"Referring expressions","total_models":1613,"url":"https://huggingface.co/datasets/JierunChen/Ref-L4"},{"bench_models":27,"description":"RefCOCO average accuracy at IoU 0.5 (val).","github":"","has_recent":0,"id":"refcoco","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RefCOCO","paper":"","placements":[{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":1,"score":92.4,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":2,"score":92.4,"version":"Qwen3 VL"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":3,"score":91.9,"version":"Intern-S1"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":4,"score":91.3,"version":"GLM 4.5V"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":5,"score":91.3,"version":"Seed 1.5"},{"company":"moondream","family":"Moondream","model":"moondream-9b-a2b","place":6,"score":91.1,"version":"Moondream 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":7,"score":91.1,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":8,"score":90.45,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":9,"score":90.3,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":10,"score":90.1,"version":"InternVL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":89.7,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":12,"score":89.6,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":13,"score":88.6,"version":"GLM 4.6V"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":14,"score":88.3,"version":"Ovis 2.6"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":15,"score":87.8,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":16,"score":87.1,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":17,"score":85.7,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":18,"score":85.6,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":19,"score":85.3,"version":"GLM 4.1V"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":20,"score":76.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":21,"score":75.8,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":22,"score":74.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":23,"score":66.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":24,"score":54.9,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":25,"score":33.6,"version":"Kimi VL"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":26,"score":30.1,"version":"Claude 4"},{"company":"stepfun","family":"Step","model":"step-3","place":27,"score":20.2,"version":"Step 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":28,"score":2.4,"version":"Gemma 3"}],"stars":"","top_company":"opengvlab","top_model":"InternVL3.5-4B","top_model_family":"Intern","top_model_id":"internvl3.5-4b","top_model_version":"InternVL 3.5","top_open_source":1,"top_percent":92.4,"topic":"Referring expressions","total_models":1613,"url":"https://huggingface.co/datasets/JierunChen/Ref-L4"},{"bench_models":4,"description":"RefCOCOg average accuracy at IoU 0.5 (val).","github":"","has_recent":0,"id":"refcocog","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RefCOCOg","paper":"","placements":[{"company":"moondream","family":"Moondream","model":"moondream-9b-a2b","place":1,"score":88.6,"version":"Moondream 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":2,"score":75.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":49.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":4,"score":26.2,"version":"Claude 4"}],"stars":"","top_company":"moondream","top_model":"Moondream-9B-A2B","top_model_family":"Moondream","top_model_id":"moondream-9b-a2b","top_model_version":"Moondream 3","top_open_source":1,"top_percent":88.6,"topic":"Referring expressions","total_models":1613,"url":"https://huggingface.co/datasets/JierunChen/Ref-L4"},{"bench_models":4,"description":"RefCOCO+ accuracy at IoU 0.5 on the val split.","github":"","has_recent":0,"id":"refcocoplus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RefCOCO+","paper":"","placements":[{"company":"moondream","family":"Moondream","model":"moondream-9b-a2b","place":1,"score":81.8,"version":"Moondream 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":2,"score":70.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":3,"score":46.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":4,"score":23.4,"version":"Claude 4"}],"stars":"","top_company":"moondream","top_model":"Moondream-9B-A2B","top_model_family":"Moondream","top_model_id":"moondream-9b-a2b","top_model_version":"Moondream 3","top_open_source":1,"top_percent":81.8,"topic":"Referring expressions","total_models":1613,"url":"https://huggingface.co/datasets/JierunChen/Ref-L4"},{"bench_models":25,"description":"Reference spatial understanding benchmark covering spatial grounding tasks.","github":"","has_recent":0,"id":"refspatialbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RefSpatialBench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":73.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":2,"score":72.1,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":3,"score":69.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":4,"score":69.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":69.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":6,"score":67.7,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":7,"score":67.5,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":65.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":9,"score":63.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":10,"score":61.4,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":11,"score":58.8,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":12,"score":54.2,"version":"Qwen3 VL"},{"company":"robobrain","family":"RoboBrain","model":"robobrain-2.0","place":13,"score":54,"version":"RoboBrain"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":14,"score":53.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":15,"score":46.6,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":36.5,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":17,"score":33.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":18,"score":30.3,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":19,"score":28.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":20,"score":27.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":21,"score":23.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":22,"score":21.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":23,"score":12.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":24,"score":9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":25,"score":4.3,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":26,"score":2.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":27,"score":2.2,"version":"Claude 4.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":73.6,"topic":"Spatial reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Safety-oriented refusal and policy adherence benchmark.","github":"","has_recent":0,"id":"refusalbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RefusalBench","paper":"","placements":[{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b-pyche","place":1,"score":72.29,"version":"Hermes 4.3"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b","place":2,"score":66.67,"version":"Hermes 4.3"},{"company":"nousresearch","family":"Hermes","model":"hermes-4-70b","place":3,"score":59.5,"version":"Hermes 4"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-70b","place":4,"score":59.2,"version":"Hermes 4.3"}],"stars":"","top_company":"nousresearch","top_model":"Hermes 4.3 36B Pyche","top_model_family":"Hermes","top_model_id":"hermes-4.3-36b-pyche","top_model_version":"Hermes 4.3","top_open_source":1,"top_percent":72.29,"topic":"Safety / refusal","total_models":1613,"url":""},{"bench_models":5,"description":"Reasoning over multimodal inputs (ReMI).","github":"","has_recent":0,"id":"remi","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ReMI","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":1,"score":67.29,"version":"Step3-VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":2,"score":63.13,"version":"MiMo VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":3,"score":60.75,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":4,"score":57.17,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":5,"score":52.65,"version":"InternVL 3.5"}],"stars":"","top_company":"stepfun","top_model":"Step3-VL-10B","top_model_family":"Step","top_model_id":"step3-vl-10b","top_model_version":"Step3-VL","top_open_source":1,"top_percent":67.29,"topic":"Multimodal reasoning","total_models":1613,"url":""},{"bench_models":16,"description":"Repository-level code comprehension and reasoning benchmark.","github":"","has_recent":0,"id":"repobench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RepoBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":1,"score":83.82,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-codex-medium","place":2,"score":67.65,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":67.35,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-codex-high","place":4,"score":66.18,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-codex-low","place":5,"score":64.71,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":64.71,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-medium","place":7,"score":63.24,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-codex","place":8,"score":61.47,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":9,"score":61.18,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":10,"score":60.29,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp-thinking","place":11,"score":59.41,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":12,"score":59.12,"version":"Claude 4.1"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":13,"score":34,"version":"Codestral"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":14,"score":28.4,"version":"DeepSeek Coder"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":15,"score":18.4,"version":"Llama 3"},{"company":"meta","family":"CodeLlama","model":"codellama-70b","place":16,"score":11.4,"version":"CodeLlama"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.5","top_model_family":"Claude","top_model_id":"claude-4.5-sonnet","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":83.82,"topic":"Code understanding","total_models":1613,"url":""},{"bench_models":7,"description":"Benchmark evaluating model ability to conduct research and synthesize findings.","github":"","has_recent":0,"id":"researchrubrics","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ResearchRubrics","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":1,"score":65.3,"version":"Step 3.5 Flash"},{"company":"zai","family":"GLM","model":"glm-4.7","place":2,"score":62,"version":"GLM 4.7"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":3,"score":60.2,"version":"MiniMax M2.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":59.5,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":5,"score":56.2,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":6,"score":55.8,"version":"DeepSeek V3.2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":7,"score":54.3,"version":"MiMo V2"}],"stars":"","top_company":"stepfun","top_model":"Step-3.5 Flash 20260204","top_model_family":"Step","top_model_id":"step-3.5-flash-20260204","top_model_version":"Step 3.5 Flash","top_open_source":1,"top_percent":65.3,"topic":"Research evaluation","total_models":1613,"url":""},{"bench_models":15,"description":"RoboSpatialHome benchmark for embodied spatial reasoning in domestic environments.","github":"","has_recent":0,"id":"robospatialhome","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RoboSpatialHome","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":1,"score":73.9,"version":"Qwen3 VL"},{"company":"robobrain","family":"RoboBrain","model":"robobrain-2.0","place":2,"score":72.4,"version":"RoboBrain"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":3,"score":70.2,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":4,"score":70.2,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":5,"score":66.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":6,"score":64.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":7,"score":62.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":8,"score":61.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5","place":9,"score":53.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":10,"score":49.1,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":47.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":12,"score":44.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":13,"score":44.6,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":14,"score":44.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":15,"score":41.2,"version":"Gemini 2.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-235B-A22B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-235b-a22b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":73.9,"topic":"Embodied spatial understanding","total_models":1613,"url":""},{"bench_models":31,"description":"Community-maintained coding evals and leaderboard by Roo Code.","github":"https://github.com/RooCodeInc/Roo-Code-Evals","has_recent":0,"id":"roo-code-evals","leaderboard":"https://roocode.com/evals","lower_is_better":false,"metric":"percent","name":"Roo Code Evals","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-mini","place":1,"score":99,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":2,"score":98,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5-medium","place":3,"score":98,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":4,"score":98,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":96,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-low","place":6,"score":95,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":7,"score":95,"version":"Claude 3.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905-groq","place":8,"score":94,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":9,"score":94,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":10,"score":91,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":11,"score":90,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":12,"score":90,"version":"Grok Code"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":13,"score":90,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":14,"score":90,"version":"Claude 3.5"},{"company":"xai","family":"Grok","model":"grok-3","place":15,"score":89,"version":"Grok 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":16,"score":89,"version":"Kimi K2"},{"company":"","family":"Sonoma","model":"sonoma-sky","place":17,"score":86,"version":"Sonoma"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":18,"score":86,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":19,"score":86,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-coder-480b-a35b-instruct","place":20,"score":84,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":21,"score":83,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":22,"score":83,"version":"GPT-4.1"},{"company":"openai","family":"o4","model":"o4-mini-high","place":23,"score":79,"version":"o4"},{"company":"","family":"Sonoma","model":"sonoma-dusk","place":24,"score":78,"version":"Sonoma"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":25,"score":78,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":26,"score":77,"version":"DeepSeek V3"},{"company":"openai","family":"o3","model":"o3-mini-high","place":27,"score":75,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-next","place":28,"score":73,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4","place":29,"score":72,"version":"Grok 4"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":30,"score":54,"version":"GLM 4.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":31,"score":52,"version":"Llama 4"}],"stars":"","top_company":"openai","top_model":"GPT-5 mini","top_model_family":"GPT","top_model_id":"gpt-5-mini","top_model_version":"GPT-5","top_open_source":0,"top_percent":99,"topic":"Code assistant eval","total_models":1613,"url":"https://roocode.com/evals"},{"bench_models":2,"description":"RULER-100 evaluation at a 1M context window.","github":"","has_recent":0,"id":"ruler-100-1m","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RULER-100 @1M","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":1,"score":86.3,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":2,"score":77.5,"version":"Qwen3"}],"stars":"","top_company":"nvidia","top_model":"NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","top_model_family":"Nemotron","top_model_id":"nvidia-nemotron-3-nano-30b-a3b-bf16","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":86.3,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":2,"description":"RULER-100 evaluation at a 256k context window.","github":"","has_recent":0,"id":"ruler-100-256k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RULER-100 @256k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":1,"score":92.9,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":2,"score":89.4,"version":"Qwen3"}],"stars":"","top_company":"nvidia","top_model":"NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","top_model_family":"Nemotron","top_model_id":"nvidia-nemotron-3-nano-30b-a3b-bf16","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":92.9,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":2,"description":"RULER-100 evaluation at a 512k context window.","github":"","has_recent":0,"id":"ruler-100-512k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RULER-100 @512k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":1,"score":91.3,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":2,"score":84,"version":"Qwen3"}],"stars":"","top_company":"nvidia","top_model":"NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","top_model_family":"Nemotron","top_model_id":"nvidia-nemotron-3-nano-30b-a3b-bf16","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":91.3,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":15,"description":"RULER benchmark at 128k context window.","github":"","has_recent":0,"id":"ruler-128k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 128k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":1,"score":96,"version":"Qwen3 Next"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":2,"score":95.6,"version":"JoyAI-LLM"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":3,"score":95.4,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":4,"score":93.9,"version":"Qwen3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":5,"score":90.2,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":6,"score":89.66,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":7,"score":89.4,"version":"MiniCPM SALA"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":8,"score":89.37,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":9,"score":89.1,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":10,"score":82.92,"version":"Nemotron 3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-nano-9b-v2","place":11,"score":78.9,"version":"Nemotron Nano v2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":12,"score":74.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":13,"score":71.74,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama3-3.2b","place":14,"score":71.3,"version":"Llama 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":15,"score":68.01,"version":"Nemotron Nano V2"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":16,"score":56.12,"version":"GLM 4.7"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":17,"score":45.09,"version":"Ministral 3"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":18,"score":36.33,"version":"Falcon H1R"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Next-80B-A3B-Instruct","top_model_family":"Qwen","top_model_id":"qwen3-next-80b-a3b-instruct","top_model_version":"Qwen3 Next","top_open_source":1,"top_percent":96,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":5,"description":"RULER benchmark at 16k context window.","github":"","has_recent":0,"id":"ruler-16k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 16k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"qwen","family":"Qwen","model":"qwen-2.5-7.6b","place":1,"score":92.2,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":2,"score":91.28,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":3,"score":90.73,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":4,"score":90.01,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":5,"score":85.88,"version":"Kanana 1.5"}],"stars":"","top_company":"qwen","top_model":"Qwen2.5 7.6B","top_model_family":"Qwen","top_model_id":"qwen-2.5-7.6b","top_model_version":"Qwen2.5","top_open_source":1,"top_percent":92.2,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":7,"description":"RULER benchmark at 1M context window.","github":"","has_recent":0,"id":"ruler-1m","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 1M","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":1,"score":94.8,"version":"Kimi Linear"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":2,"score":86.3,"version":"Nemotron 3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":3,"score":86.3,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":4,"score":84.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":5,"score":80.3,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":6,"score":77.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":7,"score":72.8,"version":"Qwen3"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-Linear-Instruct","top_model_family":"Kimi","top_model_id":"kimi-linear-instruct","top_model_version":"Kimi Linear","top_open_source":1,"top_percent":94.8,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":6,"description":"RULER benchmark at 32k context window.","github":"","has_recent":0,"id":"ruler-32k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 32k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":1,"score":96,"version":"Mistral 3"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-7.6b","place":2,"score":90.2,"version":"Qwen2.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":3,"score":88.63,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":4,"score":88.32,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":5,"score":87.92,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":6,"score":81.62,"version":"Kanana 1.5"}],"stars":"","top_company":"mistral","top_model":"Mistral Medium 3","top_model_family":"Mistral","top_model_id":"mistral-medium-3","top_model_version":"Mistral 3","top_open_source":0,"top_percent":96,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":5,"description":"RULER benchmark at 4k context window.","github":"","has_recent":0,"id":"ruler-4k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 4k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"mistral","family":"Ministral","model":"ministral-8b","place":1,"score":96,"version":"Ministral"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":2,"score":94.32,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":3,"score":93.09,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":4,"score":92.49,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":5,"score":86.39,"version":"Kanana 1.5"}],"stars":"","top_company":"mistral","top_model":"Ministral 8B","top_model_family":"Ministral","top_model_id":"ministral-8b","top_model_version":"Ministral","top_open_source":1,"top_percent":96,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":4,"description":"RULER benchmark at 512k context window.","github":"","has_recent":0,"id":"ruler-512k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 512k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":1,"score":90.9,"version":"Qwen3"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":2,"score":87.1,"version":"MiniCPM SALA"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":3,"score":86.9,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":78.4,"version":"Qwen3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-235B-A22B-Instruct-2507","top_model_family":"Qwen","top_model_id":"qwen3-235b-a22b-instruct-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":90.9,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":7,"description":"RULER benchmark at 64k context window.","github":"","has_recent":0,"id":"ruler-64k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 64k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"openbmb","family":"MiniCPM","model":"minicpm-sala","place":1,"score":92.65,"version":"MiniCPM SALA"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-v2-9b","place":2,"score":88.77,"version":"Nemotron Nano V2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":3,"score":87.5,"version":"Nemotron 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":4,"score":84.7,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":5,"score":80.53,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-r-8b","place":6,"score":70.66,"version":"Ministral 3"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":7,"score":56.5,"version":"Falcon H1R"}],"stars":"","top_company":"openbmb","top_model":"MiniCPM-SALA","top_model_family":"MiniCPM","top_model_id":"minicpm-sala","top_model_version":"MiniCPM SALA","top_open_source":1,"top_percent":92.65,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":5,"description":"RULER benchmark at 8k context window.","github":"","has_recent":0,"id":"ruler-8k","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Ruler 8k","paper":"https://arxiv.org/abs/2404.06654","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":1,"score":93.8,"version":"Llama 3.1"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":2,"score":92.29,"version":"Kanana 2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":3,"score":92.16,"version":"Qwen3"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":4,"score":92.14,"version":"Kanana 2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":5,"score":90.16,"version":"Kanana 1.5"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 8B Base","top_model_family":"Llama","top_model_id":"llama-3.1-8b-base","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":93.8,"topic":"Long-context eval","total_models":1613,"url":"https://github.com/allenai/ruler"},{"bench_models":8,"description":"Real-world search benchmark evaluating retrieval and reasoning.","github":"","has_recent":0,"id":"rw-search","leaderboard":"","lower_is_better":false,"metric":"percent","name":"RW Search","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":1,"score":82,"version":"GPT-5.2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":2,"score":79.5,"version":"LongCat-Flash"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":3,"score":75.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":74.5,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":5,"score":74,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":6,"score":69,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":7,"score":63,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":8,"score":20.5,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 Thinking XHigh","top_model_family":"GPT","top_model_id":"gpt-5.2-thinking-xhigh","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":82,"topic":"Agentic search","total_models":1613,"url":""},{"bench_models":4,"description":"Safety Alignment and Dangerous-behavior benchmark evaluating harmful assistance and refusal consistency.","github":"https://github.com/allenai/salad-bench","has_recent":0,"id":"salad-bench","leaderboard":"","lower_is_better":true,"metric":"percent","name":"SALAD-Bench","paper":"https://arxiv.org/abs/2410.08628","placements":[{"company":"IBM","family":"Granite","model":"granite-4.0-h-micro","place":1,"score":96.77,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-tiny","place":2,"score":96.77,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-micro","place":3,"score":97.06,"version":"Granite 4.0"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":4,"score":97.3,"version":"Granite 4.0"}],"stars":"","top_company":"IBM","top_model":"Granite-4.0-H-Micro","top_model_family":"Granite","top_model_id":"granite-4.0-h-micro","top_model_version":"Granite 4.0","top_open_source":1,"top_percent":96.77,"topic":"Safety alignment","total_models":1613,"url":"https://github.com/allenai/salad-bench"},{"bench_models":5,"description":"SVG Arena benchmark for icon generation evaluation.","github":"","has_recent":0,"id":"sarena-icon","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SArena (Icon)","paper":"","placements":[{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":1,"score":83.5,"version":"Intern-S1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":82.6,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":77.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":4,"score":76.3,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":55.5,"version":"GPT-5.2"}],"stars":"","top_company":"internlm","top_model":"Intern-S1-Pro","top_model_family":"Intern","top_model_id":"intern-s1-pro","top_model_version":"Intern-S1","top_open_source":1,"top_percent":83.5,"topic":"SVG generation","total_models":1613,"url":""},{"bench_models":3,"description":"Scale AI Multi Challenge crowd-evaluated instruction following benchmark.","github":"","has_recent":0,"id":"scale-ai-multi-challenge","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Scale AI Multi Challenge","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":1,"score":44.8,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":2,"score":38.5,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":3,"score":33.8,"version":"GPT-OSS"}],"stars":"","top_company":"qwen","top_model":"Qwen3-30B-A3B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-30b-a3b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":44.8,"topic":"Chat \u0026 instruction following","total_models":1613,"url":""},{"bench_models":71,"description":"SciCode subset score (sub).","github":"","has_recent":0,"id":"scicode","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SciCode (sub)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":59,"version":"Gemini 3.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":56.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":56,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":4,"score":52.1,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":5,"score":52,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":6,"score":52,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":52,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":50,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":9,"score":49.5,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":10,"score":48.7,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":11,"score":47,"version":"Claude 4.6"},{"company":"xai","family":"Grok","model":"grok-4","place":12,"score":45.7,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":45,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":14,"score":44.8,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":15,"score":44.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":16,"score":44.7,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":17,"score":44.2,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":18,"score":42.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":19,"score":42.9,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":20,"score":42.8,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-instruct-2507","place":21,"score":42.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":22,"score":42.4,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3-high","place":23,"score":41.7,"version":"o3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":24,"score":41,"version":"MiniMax M2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":25,"score":40.6,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":26,"score":40.3,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":27,"score":40,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":28,"score":40,"version":"Claude 4"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-32b","place":29,"score":39.6,"version":"Nemotron OpenReasoning"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":30,"score":39.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":31,"score":39.3,"version":"Qwen3"},{"company":"mbzuai","family":"K2","model":"k2-think","place":32,"score":39.2,"version":"K2-THINK"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":33,"score":39.2,"version":"Magistral 1.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":34,"score":39,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":35,"score":38.9,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":36,"score":38.8,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":37,"score":38.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":38,"score":38.8,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":39,"score":38.2,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":40,"score":37.9,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":41,"score":37.7,"version":"DeepSeek V3.2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":42,"score":37.1,"version":"Nemotron"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":43,"score":36.9,"version":"QwQ"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":44,"score":36.7,"version":"DeepSeek R1"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":45,"score":36.46,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":46,"score":36.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":47,"score":36.2,"version":"GPT-OSS"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":48,"score":36,"version":"MiniMax M2"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":49,"score":35.4,"version":"GPT-OSS"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":50,"score":35.2,"version":"Magistral 1.2"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":51,"score":34.9,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5","place":52,"score":34.8,"version":"GLM 4.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":53,"score":34.8,"version":"Apriel 1.5"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":54,"score":34.4,"version":"EXAONE 4.0"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":55,"score":34,"version":"Llama Nemotron v1.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":56,"score":34,"version":"GPT-OSS"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":57,"score":33.3,"version":"Nemotron 3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":58,"score":33.1,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":59,"score":33,"version":"Qwen3"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":60,"score":33,"version":"Mi:dm K 2.5"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-14b","place":61,"score":32.4,"version":"Nemotron OpenReasoning"},{"company":"openai","family":"GPT","model":"gpt-4-turbo","place":62,"score":32,"version":"GPT-4"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":63,"score":31.9,"version":"Apriel 1.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":64,"score":30.7,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":65,"score":30.6,"version":"GLM 4.5"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":66,"score":29.8,"version":"Phi-4"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":67,"score":28.5,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":68,"score":28.3,"version":"Falcon H1R"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":69,"score":28.3,"version":"Qwen3"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":70,"score":28,"version":"HyperCLOVA X"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":71,"score":26.1,"version":"Nemotron-H"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":72,"score":25.6,"version":"Qwen3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":73,"score":22.92,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":74,"score":22.92,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":75,"score":22.2,"version":"DeepSeek R1"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-7b","place":76,"score":20.3,"version":"Nemotron OpenReasoning"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":77,"score":15.11,"version":"GLM 4.7"},{"company":"nvidia","family":"Nemotron","model":"openreasoning-nemotron-1.5b","place":78,"score":1,"version":"Nemotron OpenReasoning"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":59,"topic":"Code","total_models":1613,"url":""},{"bench_models":22,"description":"SciCode main score.","github":"","has_recent":0,"id":"scicode-main","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SciCode (main)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":15.4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":12.4,"version":"GPT-5"},{"company":"mbzuai","family":"K2","model":"k2-think","place":3,"score":12,"version":"K2-THINK"},{"company":"openai","family":"o3","model":"o3-high","place":4,"score":11.9,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":5,"score":11.7,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":6,"score":11.5,"version":"DeepSeek R1"},{"company":"qwen","family":"QwQ","model":"qwq-32b","place":7,"score":11.5,"version":"QwQ"},{"company":"nvidia","family":"Nemotron","model":"nemotron-32b","place":8,"score":11.4,"version":"Nemotron"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":9,"score":11,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":10,"score":10.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":11,"score":9.2,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":12,"score":9,"version":"GPT-OSS"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":13,"score":8.2,"version":"Apriel 1.5"},{"company":"microsoft","family":"Phi 4","model":"phi-4-reasoning-plus","place":14,"score":7.2,"version":"Phi-4"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":15,"score":6.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":16,"score":6.2,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":17,"score":4.8,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":18,"score":4.6,"version":"Nemotron-H"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":19,"score":3.9,"version":"Falcon H1R"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":20,"score":3.08,"version":"JoyAI-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":21,"score":3.08,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":22,"score":3.08,"version":"GLM 4.7"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528-qwen3-8b","place":23,"score":2.6,"version":"DeepSeek R1"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":15.4,"topic":"Code","total_models":1613,"url":""},{"bench_models":30,"description":"Multiple-choice science questions with images, diagrams, and text context.","github":"","has_recent":0,"id":"scienceqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ScienceQA","paper":"","placements":[{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":1,"score":96.7,"version":"FastVLM"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":2,"score":96.2,"version":"Manzano"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":3,"score":94.4,"version":"FastVLM"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":4,"score":92.9,"version":"Manzano"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":5,"score":91.9,"version":"MM1.5"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":6,"score":90.8,"version":"Phi-3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":7,"score":90.7,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":88.4,"version":"Gemini 2.5"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":9,"score":88.3,"version":"BLIP-3"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":10,"score":85.8,"version":"MM1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":11,"score":85.7,"version":"Gemini 1.5"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":12,"score":85.6,"version":"Cambrian"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":13,"score":85.2,"version":"FastVLM"},{"company":"bytedance","family":"LLaVA","model":"llava-next-34b","place":14,"score":81.8,"version":"LLaVA-NeXT"},{"company":"apple","family":"MM1","model":"mm1-30b","place":15,"score":81,"version":"MM1"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":16,"score":80.7,"version":"MiniCPM V 2.0"},{"company":"baai","family":"Bunny","model":"bunny-4b","place":17,"score":78.3,"version":"Bunny"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":18,"score":69,"version":"VILA 1.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":19,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":20,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":21,"score":0,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":22,"score":0,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":23,"score":0,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":24,"score":0,"version":"InternVL 2.5"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":25,"score":0,"version":"Janus Pro"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":26,"score":0,"version":"BLIP-3O"},{"company":"baai","family":"Emu","model":"emu3-8b","place":27,"score":0,"version":"Emu3"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":28,"score":0,"version":"Janus Pro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":29,"score":0,"version":"Qwen2.5 Omni"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":30,"score":0,"version":"Bagel"}],"stars":"","top_company":"apple","top_model":"FastVLM-7B","top_model_family":"FastVLM","top_model_id":"fastvlm-7b","top_model_version":"FastVLM","top_open_source":1,"top_percent":96.7,"topic":"Science QA (multimodal)","total_models":1613,"url":""},{"bench_models":8,"description":"Multiple choice science questions.","github":"","has_recent":0,"id":"sciq","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SciQ","paper":"","placements":[{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":1,"score":92.9,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":2,"score":91.1,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":3,"score":88.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":4,"score":87,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":5,"score":86.5,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":6,"score":82.6,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":7,"score":72,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":8,"score":60.6,"version":"Pythia"}],"stars":"","top_company":"eleutherai","top_model":"Pythia 12B","top_model_family":"Pythia","top_model_id":"pythia-12b","top_model_version":"Pythia","top_open_source":1,"top_percent":92.9,"topic":"Science QA","total_models":1613,"url":"https://allenai.org/data/sciq"},{"bench_models":5,"description":"Scientific reasoning benchmark evaluating multimodal AI models on scientific tasks.","github":"","has_recent":0,"id":"scireasoner","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SciReasoner","paper":"","placements":[{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":1,"score":55.5,"version":"Intern-S1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":2,"score":15.3,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":14.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":13.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":11.9,"version":"Qwen3 VL"}],"stars":"","top_company":"internlm","top_model":"Intern-S1-Pro","top_model_family":"Intern","top_model_id":"intern-s1-pro","top_model_version":"Intern-S1","top_open_source":1,"top_percent":55.5,"topic":"Scientific reasoning","total_models":1613,"url":""},{"bench_models":3,"description":"SciRes FrontierMath benchmark covering tiers 1-3.","github":"","has_recent":0,"id":"scires-frontiermath-tier-1-3","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SciRes FrontierMath Tier 1-3","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":1,"score":40.3,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":37.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1-thinking","place":3,"score":31,"version":"GPT-5.1"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 Thinking","top_model_family":"GPT","top_model_id":"gpt-5.2-thinking","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":40.3,"topic":"Math (frontier)","total_models":1613,"url":""},{"bench_models":3,"description":"SciRes FrontierMath benchmark covering tier 4.","github":"","has_recent":0,"id":"scires-frontiermath-tier-4","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SciRes FrontierMath Tier 4","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":18.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":2,"score":14.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.1-thinking","place":3,"score":12.5,"version":"GPT-5.1"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":18.8,"topic":"Math (frontier)","total_models":1613,"url":""},{"bench_models":12,"description":"Complex ScreenQA benchmark accuracy.","github":"","has_recent":0,"id":"screenqa-complex","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ScreenQA Complex","paper":"","placements":[{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":1,"score":87.1,"version":"Holo1.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":2,"score":83.3,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":3,"score":83.2,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":4,"score":83.2,"version":"Qwen2.5 VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":5,"score":82.3,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":6,"score":81.4,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":7,"score":81.1,"version":"Qwen2.5 VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":8,"score":79.2,"version":"UI-TARS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":9,"score":76,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":10,"score":75.7,"version":"Claude 4"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":11,"score":65.1,"version":"Holo1"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":12,"score":53.5,"version":"Holo1"}],"stars":"","top_company":"hcompany","top_model":"Holo1.5-72B","top_model_family":"Holo","top_model_id":"holo1.5-72b","top_model_version":"Holo1.5","top_open_source":1,"top_percent":87.1,"topic":"GUI QA","total_models":1613,"url":""},{"bench_models":12,"description":"Short-form ScreenQA benchmark accuracy.","github":"","has_recent":0,"id":"screenqa-short","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ScreenQA Short","paper":"","placements":[{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":1,"score":91.9,"version":"Holo1.5"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":2,"score":91,"version":"Holo1.5"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":3,"score":88.7,"version":"UI-TARS"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":4,"score":88.6,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":5,"score":87.9,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":6,"score":87.9,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":7,"score":87,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":8,"score":87,"version":"Qwen2.5 VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":9,"score":86.3,"version":"UI-Venus"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":10,"score":86,"version":"Qwen2.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":11,"score":83.3,"version":"Holo1"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":12,"score":78.3,"version":"Holo1"}],"stars":"","top_company":"hcompany","top_model":"Holo1.5-72B","top_model_family":"Holo","top_model_id":"holo1.5-72b","top_model_version":"Holo1.5","top_open_source":1,"top_percent":91.9,"topic":"GUI QA","total_models":1613,"url":""},{"bench_models":14,"description":"Center accuracy on ScreenSpot.","github":"","has_recent":0,"id":"screenspot","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ScreenSpot","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":1,"score":95.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":2,"score":95.4,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":95.2,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":4,"score":94.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":5,"score":94.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":6,"score":94,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":7,"score":91.4,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":8,"score":87.3,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":9,"score":87.2,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":10,"score":87.1,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":11,"score":84.7,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":12,"score":79.5,"version":"InternVL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":13,"score":70.2,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":14,"score":66.3,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL 32B Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-32b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":95.8,"topic":"Screen UI locators","total_models":1613,"url":""},{"bench_models":52,"description":"Average center accuracy on ScreenSpot-Pro.","github":"","has_recent":0,"id":"screenspot-pro","leaderboard":"https://gui-agent.github.io/grounding-leaderboard/","lower_is_better":false,"metric":"percent","name":"ScreenSpot-Pro","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":1,"score":86.3,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":73.1,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":72.7,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":70.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":5,"score":70.3,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":6,"score":69.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":7,"score":68.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":8,"score":65.6,"version":"Qwen3.5"},{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":9,"score":63.25,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":10,"score":62,"version":"Qwen3 VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":11,"score":61.9,"version":"UI-Venus"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":12,"score":61.8,"version":"Qwen3 VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5","place":13,"score":61.6,"version":"UI-TARS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":14,"score":61,"version":"Qwen3 VL"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":15,"score":60.9,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":16,"score":60.9,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":17,"score":59.5,"version":"Qwen3 VL"},{"company":"salesforce","family":"GTA1","model":"gta1-72b","place":18,"score":58.4,"version":"GTA1"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":19,"score":57.94,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":20,"score":57.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":21,"score":55.6,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":22,"score":54.6,"version":"Qwen3 VL"},{"company":"salesforce","family":"GTA1","model":"gta1-32b","place":23,"score":53.6,"version":"GTA1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":24,"score":53.3,"version":"Qwen2.5 VL"},{"company":"lenovo","family":"TianXi","model":"tianxi-action-7b","place":25,"score":51.9,"version":"TianXi Action"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":26,"score":51.55,"version":"Step3-VL"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":27,"score":51.49,"version":"Holo1.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":28,"score":51,"version":"Kimi VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":29,"score":50.8,"version":"UI-Venus"},{"company":"","family":"V2P","model":"v2p-7b","place":30,"score":50.6,"version":"V2P"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":31,"score":48.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":32,"score":46.6,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":33,"score":45.7,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":34,"score":45.68,"version":"GLM 4.6V"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":35,"score":44.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":36,"score":43.6,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":37,"score":43.6,"version":"GPT-5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":38,"score":41.9,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":39,"score":39.9,"version":"MiMo VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":40,"score":39,"version":"UI-TARS"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":41,"score":36.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":42,"score":36.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":43,"score":36.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":44,"score":36.2,"version":"Claude 4.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":45,"score":34.84,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":46,"score":29.3,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":47,"score":29,"version":"Qwen2.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":48,"score":26.06,"version":"Holo1"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":49,"score":23.66,"version":"Holo1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":50,"score":19.1,"version":"Claude 4"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":51,"score":15.39,"version":"InternVL 3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":52,"score":11.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":53,"score":3.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":54,"score":3.5,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":55,"score":3.5,"version":"Claude 4.5"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 Extra High","top_model_family":"GPT","top_model_id":"gpt-5.2-extra-high","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":86.3,"topic":"Screen UI locators","total_models":1613,"url":""},{"bench_models":25,"description":"Center accuracy on ScreenSpot-v2.","github":"","has_recent":0,"id":"screenspot-v2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ScreenSpot-v2","paper":"","placements":[{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":1,"score":95.3,"version":"UI-Venus"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":94.7,"version":"Gemini 3"},{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":3,"score":94.41,"version":"Holo1.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":4,"score":94.1,"version":"UI-Venus"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":5,"score":94,"version":"UI-TARS"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-thinking","place":6,"score":93.6,"version":"Qwen3 VL"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":7,"score":93.6,"version":"Intern-S1"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":8,"score":93.31,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":9,"score":93.3,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":10,"score":93,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":11,"score":92.8,"version":"Qwen3 VL"},{"company":"stepfun","family":"Step","model":"step3-vl-10b","place":12,"score":92.61,"version":"Step3-VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":13,"score":92.4,"version":"Kimi K2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":14,"score":92.14,"version":"GLM 4.6V"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":15,"score":91.66,"version":"Holo1.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl-2508","place":16,"score":90.82,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":17,"score":90.5,"version":"MiMo VL"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":18,"score":89.85,"version":"Holo1"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":19,"score":89.5,"version":"MiMo VL"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":20,"score":88.91,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":21,"score":88,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":22,"score":84.02,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":23,"score":81.4,"version":"InternVL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":24,"score":80,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":25,"score":49.4,"version":"GPT-5.2"}],"stars":"","top_company":"antgroup","top_model":"UI-Venus 72B","top_model_family":"UI-Venus","top_model_id":"ui-venus-72b","top_model_version":"UI-Venus","top_open_source":1,"top_percent":95.3,"topic":"Screen UI locators","total_models":1613,"url":""},{"bench_models":24,"description":"Evaluation of multi-step browsing agents on search, evidence gathering, and synthesis tasks.","github":"","has_recent":0,"id":"seal-0","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SEAL-0","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":1,"score":57.4,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":2,"score":57.4,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":3,"score":56.3,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":4,"score":53.4,"version":"Claude 4.5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-235b","place":5,"score":52.7,"version":"MiroThinker v1.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":6,"score":51.4,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":51.4,"version":"GPT-5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.0-72b","place":8,"score":51,"version":"MiroThinker v1.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":9,"score":49.5,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":10,"score":47.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":47.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":12,"score":47.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":13,"score":46.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":14,"score":46.9,"version":"Qwen3.5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-30b","place":15,"score":46.2,"version":"MiroThinker v1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":45.5,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":17,"score":45.1,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":18,"score":45,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":19,"score":45,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":20,"score":44.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":21,"score":41.4,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":22,"score":38.5,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":23,"score":34.2,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":24,"score":25.2,"version":"Kimi K2"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2.5 Thinking","top_model_family":"Kimi","top_model_id":"kimi-k2.5-thinking","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":57.4,"topic":"Agentic web search","total_models":1613,"url":""},{"bench_models":6,"description":"Benchmark evaluating secure code generation capabilities.","github":"","has_recent":0,"id":"seccodebench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SecCodeBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":68.7,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":68.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":68.3,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":62.4,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":5,"score":61.3,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":6,"score":57.5,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":68.7,"topic":"Secure code generation","total_models":1613,"url":""},{"bench_models":12,"description":"SEED-Bench-2-Plus overall accuracy.","github":"","has_recent":0,"id":"seed-bench-2-plus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SEED-Bench-2-Plus","paper":"https://arxiv.org/abs/2404.16790","placements":[{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":1,"score":72.9,"version":"Claude 3.7"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":2,"score":72.4,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":3,"score":71.9,"version":"MiMo VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":4,"score":71.1,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":5,"score":70.7,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":6,"score":69.7,"version":"InternVL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":7,"score":68,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":8,"score":67.3,"version":"Qwen3 VL"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":9,"score":67.2,"version":"jina-VLM"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":10,"score":66.3,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":11,"score":64.6,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":12,"score":62.4,"version":"Qwen2 VL"}],"stars":"","top_company":"anthropic","top_model":"Claude 3.7 Sonnet","top_model_family":"Claude","top_model_id":"claude-3.7-sonnet","top_model_version":"Claude 3.7","top_open_source":0,"top_percent":72.9,"topic":"Multimodal evaluation","total_models":1613,"url":"https://huggingface.co/datasets/AILab-CVC/SEED-Bench-2-plus"},{"bench_models":31,"description":"SEED-Bench image-only subset (SEED-Bench-Img).","github":"","has_recent":0,"id":"seed-bench-img","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SEED-Bench-Img","paper":"","placements":[{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":1,"score":78.5,"version":"Bagel"},{"company":"openai","family":"GPT","model":"gpt-4o","place":2,"score":77.1,"version":"GPT-4o"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":3,"score":76,"version":"Manzano"},{"company":"bytedance","family":"LLaVA","model":"llava-next-34b","place":4,"score":75.9,"version":"LLaVA-NeXT"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":5,"score":75.4,"version":"FastVLM"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":6,"score":75.3,"version":"Cambrian"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":7,"score":75,"version":"MM1.5"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":8,"score":74.3,"version":"Manzano"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":9,"score":74.2,"version":"FastVLM"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":10,"score":74.1,"version":"Qwen2.5 Omni"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":11,"score":73.8,"version":"BLIP-3O"},{"company":"baai","family":"Bunny","model":"bunny-4b","place":12,"score":72.5,"version":"Bunny"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":13,"score":72.4,"version":"MM1.5"},{"company":"salesforce","family":"BLIP","model":"blip-3-4b","place":14,"score":72.2,"version":"BLIP-3"},{"company":"apple","family":"MM1","model":"mm1-30b","place":15,"score":72.1,"version":"MM1"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":16,"score":72.1,"version":"Janus Pro"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":17,"score":71.8,"version":"Phi-3"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":18,"score":71,"version":"FastVLM"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":19,"score":68.3,"version":"Janus Pro"},{"company":"baai","family":"Emu","model":"emu3-8b","place":20,"score":68.2,"version":"Emu3"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":21,"score":67.9,"version":"VILA 1.5"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":22,"score":67.1,"version":"MiniCPM V 2.0"},{"company":"harmonai","family":"Harmon","model":"harmon-1.5b","place":23,"score":67.1,"version":"Harmon-1.5B"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":24,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":25,"score":0,"version":"InternVL 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":26,"score":0,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":27,"score":0,"version":"InternVL 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":28,"score":0,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":29,"score":0,"version":"InternVL 2.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":30,"score":0,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":31,"score":0,"version":"Gemini 2.5"}],"stars":"","top_company":"jondurbin","top_model":"Bagel 14B","top_model_family":"Bagel","top_model_id":"bagel-14b","top_model_version":"Bagel","top_open_source":1,"top_percent":78.5,"topic":"Multimodal image understanding","total_models":1613,"url":""},{"bench_models":5,"description":"SEED-Bench comprehensive multimodal understanding benchmark evaluating generative comprehension across multiple dimensions.","github":"https://github.com/AILab-CVC/SEED-Bench","has_recent":0,"id":"seedbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SEED-Bench","paper":"https://arxiv.org/abs/2307.16125","placements":[{"company":"liquidai","family":"LFM","model":"lfm2-vl-3b","place":1,"score":76.55,"version":"LFM2-VL"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":2,"score":75.41,"version":"InternVL 3.5"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":3,"score":74.95,"version":"InternVL 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":4,"score":73.88,"version":"Qwen2.5 VL"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":5,"score":71.3,"version":"SmolVLM2"}],"stars":"","top_company":"liquidai","top_model":"LFM2-VL-3B","top_model_family":"LFM","top_model_id":"lfm2-vl-3b","top_model_version":"LFM2-VL","top_open_source":1,"top_percent":76.55,"topic":"Multimodal evaluation","total_models":1613,"url":"https://huggingface.co/datasets/AILab-CVC/SEED-Bench"},{"bench_models":10,"description":"Structured factual evaluation for multimodal models.","github":"","has_recent":0,"id":"sfe","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SFE","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":61.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":58.9,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":53.7,"version":"Kimi K2.5"},{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":4,"score":52.7,"version":"Intern-S1"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":5,"score":51.2,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":6,"score":50.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":47.7,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":47.5,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":46,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":10,"score":44.4,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":11,"score":41.4,"version":"Qwen3 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":61.9,"topic":"Multimodal reasoning","total_models":1613,"url":""},{"bench_models":12,"description":"Success rate on the Showdown UI interaction benchmark.","github":"","has_recent":0,"id":"showdown","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Showdown","paper":"","placements":[{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":1,"score":76.84,"version":"Holo1.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":2,"score":75.58,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":3,"score":72.17,"version":"Holo1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":4,"score":72,"version":"Claude 4"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":5,"score":67.5,"version":"Holo1.5"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":6,"score":67.32,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":7,"score":64.27,"version":"Holo1"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":8,"score":59.96,"version":"Holo1"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":9,"score":58,"version":"UI-TARS"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":10,"score":52,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":11,"score":50.3,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":12,"score":41,"version":"Qwen2.5 VL"}],"stars":"","top_company":"hcompany","top_model":"Holo1.5-72B","top_model_family":"Holo","top_model_id":"holo1.5-72b","top_model_version":"Holo1.5","top_open_source":1,"top_percent":76.84,"topic":"GUI agents","total_models":1613,"url":""},{"bench_models":6,"description":"Single-turn instruction following benchmark.","github":"","has_recent":0,"id":"sifo","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SIFO","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":1,"score":66.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":2,"score":50.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":3,"score":47.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":4,"score":45.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":5,"score":44.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":6,"score":42.7,"version":"Qwen3 VL"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-30B-A3B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-30b-a3b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":66.9,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":6,"description":"Multi-turn SIFO benchmark for sustained instruction adherence.","github":"","has_recent":0,"id":"sifo-multiturn","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SIFO Multiturn","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":1,"score":60.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":2,"score":38.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":3,"score":37.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":4,"score":36.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":5,"score":35.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":6,"score":35.1,"version":"Qwen3 VL"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-30B-A3B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-30b-a3b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":60.3,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":111,"description":"Simple question answering benchmark.","github":"","has_recent":0,"id":"simpleqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SimpleQA","paper":"","placements":[{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":1,"score":97.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":2,"score":96.8,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":3,"score":93.4,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":4,"score":93.4,"version":"DeepSeek V3.1"},{"company":"menlo","family":"Jan","model":"jan-v1","place":5,"score":91.1,"version":"Jan v1"},{"company":"menlo","family":"Jan","model":"jan-v1-2509","place":6,"score":90.7,"version":"Jan v1"},{"company":"perplexity","family":"Perplexity","model":"perplexity-pro","place":7,"score":90.6,"version":"Perplexity"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":8,"score":86.5,"version":"Qwen3"},{"company":"menlo","family":"Jan","model":"jan-nano-128k","place":9,"score":83.2,"version":"Jan nano"},{"company":"menlo","family":"Jan","model":"jan-v1-edge","place":10,"score":83,"version":"Jan v1"},{"company":"menlo","family":"Jan","model":"jan-nano","place":11,"score":80.7,"version":"Jan nano"},{"company":"menlo","family":"Jan","model":"jan-nano-yarn","place":12,"score":79.7,"version":"Jan nano"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":72.1,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen-1.7b-thinking","place":14,"score":69.14,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":15,"score":62.5,"version":"GPT-4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":16,"score":55.1,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":17,"score":54.5,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":54,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-2025-08-07","place":19,"score":51.1,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":20,"score":50.5,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":21,"score":50.4,"version":"Qwen3"},{"company":"xai","family":"Grok","model":"grok-4-0709","place":22,"score":50.3,"version":"Grok 4"},{"company":"openai","family":"o3","model":"o3","place":23,"score":49.4,"version":"o3"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":24,"score":47.2,"version":"o1"},{"company":"openai","family":"o1","model":"o1","place":25,"score":47,"version":"o1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":26,"score":45.9,"version":"ERNIE 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":27,"score":44.3,"version":"Gemini 2.0"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":28,"score":42.3,"version":"GPT-4.1"},{"company":"xai","family":"Grok","model":"grok-3","place":29,"score":41.3,"version":"Grok 3"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":30,"score":40.9,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":31,"score":40.2,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-08-06","place":32,"score":38.4,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o","place":33,"score":38.2,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":34,"score":37.6,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-3-7-sonnet-20250219","place":35,"score":37.5,"version":"Claude 3.7"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20241022","place":36,"score":36.2,"version":"Claude 3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":37,"score":35.3,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":38,"score":34.9,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":39,"score":32.3,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":40,"score":31.9,"version":"DeepSeek R1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":41,"score":31,"version":"Kimi K2"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":42,"score":30.94,"version":"Cogito 671B v2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":43,"score":30.1,"version":"DeepSeek R1"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":44,"score":30,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash","place":45,"score":29.9,"version":"Gemini 2.0"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-mid-2601","place":46,"score":29.7,"version":"Kanana 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":47,"score":29.5,"version":"DeepSeek V3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":48,"score":29.3,"version":"Claude 4.5"},{"company":"kakao","family":"Kanana","model":"kanana-2-30b-a3b-base-2601","place":49,"score":29.13,"version":"Kanana 2"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":50,"score":28.2,"version":"Gemini 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":51,"score":27.8,"version":"DeepSeek R1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-preview-05-20","place":52,"score":27.8,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":53,"score":27.7,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":54,"score":27.3,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":55,"score":27,"version":"Qwen3 VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":56,"score":27,"version":"DeepSeek V3.2"},{"company":"kakao","family":"Kanana","model":"kanana-1.5-32.5b-base","place":57,"score":26.95,"version":"Kanana 1.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":58,"score":26.6,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":59,"score":26.47,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":60,"score":26.3,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":61,"score":26.2,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":62,"score":26.1,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":63,"score":24.9,"version":"Gemini 1.5"},{"company":"mistral","family":"Mistral","model":"mistral-large-2411","place":64,"score":24.2,"version":"Mistral Large"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":65,"score":23.9,"version":"Qwen3 VL"},{"company":"mistral","family":"Mistral","model":"mistral-large-3-base","place":66,"score":23.8,"version":"Mistral 3"},{"company":"xai","family":"Grok","model":"grok-2-1212","place":67,"score":23.6,"version":"Grok 2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":68,"score":23.3,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":69,"score":23.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":70,"score":22.8,"version":"Claude 4"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":71,"score":20.6,"version":"MiMo V2"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":72,"score":20.5,"version":"o4"},{"company":"xai","family":"Grok","model":"grok-3-mini","place":73,"score":20.4,"version":"Grok 3"},{"company":"mistral","family":"Mixtral","model":"open-mixtral-8x22b-2404","place":74,"score":19.3,"version":"Mixtral"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":75,"score":18.5,"version":"MiniMax M1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":76,"score":17.9,"version":"MiniMax M1"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-70b","place":77,"score":17.9,"version":"Hermes 4.3"},{"company":"openai","family":"o3","model":"o3-mini-2025-01-31","place":78,"score":16.3,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":79,"score":15.9,"version":"Claude 4"},{"company":"openai","family":"o3","model":"o3-mini","place":80,"score":15,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":81,"score":13.2,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":82,"score":12.9,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":83,"score":12.6,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":84,"score":12.4,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-3-5-haiku-20241022","place":85,"score":12.2,"version":"Claude 3.5"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":86,"score":12.1,"version":"Mistral 3.2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-1106","place":87,"score":11.9,"version":"GPT-3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":88,"score":11,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-2509","place":89,"score":10.7,"version":"Magistral 1.2"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":90,"score":10.43,"version":"Mistral 3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":91,"score":10.1,"version":"Kimi Linear"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":92,"score":10,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":93,"score":10,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":94,"score":10,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-4o-mini-2024-07-18","place":95,"score":9.3,"version":"GPT-4o"},{"company":"google","family":"Gemma","model":"gemma-2-27b","place":96,"score":9.2,"version":"Gemma 2"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-002","place":97,"score":9.2,"version":"Gemini 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":98,"score":9.1,"version":"Qwen3"},{"company":"openai","family":"o1","model":"o1-mini-2024-09-12","place":99,"score":8.9,"version":"o1"},{"company":"arcee","family":"Trinity","model":"trinity-mini","place":100,"score":8.9,"version":"Trinity"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":101,"score":8.6,"version":"Gemini 1.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-h-small","place":102,"score":8.6,"version":"Granite 4.0"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":103,"score":6.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":104,"score":6.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-12b-it","place":105,"score":6.1,"version":"Gemma 3"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b-pyche","place":106,"score":6,"version":"Hermes 4.3"},{"company":"mistral","family":"Ministral","model":"ministral-8b-2410","place":107,"score":5.9,"version":"Ministral"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":108,"score":5.6,"version":"Moonlight"},{"company":"nousresearch","family":"Hermes","model":"hermes-4.3-36b","place":109,"score":5.6,"version":"Hermes 4.3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":110,"score":5.5,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":111,"score":5.5,"version":"GPT-OSS"},{"company":"google","family":"Gemma","model":"gemma-2-9b","place":112,"score":5.3,"version":"Gemma 2"},{"company":"mistral","family":"Codestral","model":"codestral-2501","place":113,"score":5.3,"version":"Codestral"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-001","place":114,"score":5,"version":"Gemini 1.5"},{"company":"mistral","family":"Ministral","model":"ministral-3b-2410","place":115,"score":4.8,"version":"Ministral"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":116,"score":4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":117,"score":4,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":118,"score":3.8,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":119,"score":2.8,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":120,"score":2.2,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":121,"score":2.2,"version":"Gemma 3"}],"stars":"","top_company":"deepseek","top_model":"DeepSeek V3.2-Exp","top_model_family":"DeepSeek","top_model_id":"deepseek-v3.2-exp","top_model_version":"DeepSeek V3.2","top_open_source":1,"top_percent":97.1,"topic":"QA","total_models":1613,"url":""},{"bench_models":19,"description":"Verified SimpleQA variant for parametric knowledge accuracy.","github":"","has_recent":0,"id":"simpleqa-verified","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SimpleQA Verified","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":72.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":68.7,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":65.4,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":54.5,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":48.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":6,"score":38,"version":"GPT-5.2"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":7,"score":37.4,"version":"Ling 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":8,"score":36.8,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":9,"score":36,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":29.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":11,"score":29.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":29.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":13,"score":28.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":14,"score":26,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":15,"score":25.4,"version":"Kimi K2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":16,"score":24,"version":"Seed 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":17,"score":23.7,"version":"DeepSeek V3.2"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":18,"score":19.5,"version":"Grok 4.1"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":19,"score":19.3,"version":"Ling 1T"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":20,"score":18.9,"version":"Seed 2.0"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":72.1,"topic":"QA","total_models":1613,"url":""},{"bench_models":33,"description":"Lightweight visual question answering set with everyday scenes.","github":"","has_recent":0,"id":"simplevqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SimpleVQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":73.2,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":2,"score":71.2,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":71.2,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":69.7,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":5,"score":69.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":6,"score":67.1,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":65.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":65.4,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":9,"score":65.4,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":10,"score":63.4,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":62,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":12,"score":61.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":13,"score":61.7,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":14,"score":61.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":15,"score":61.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":16,"score":58.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":17,"score":58.2,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":18,"score":57.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":19,"score":56.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":20,"score":56.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":21,"score":56.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":22,"score":56.7,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":23,"score":56.1,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":24,"score":56,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":25,"score":55.8,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":26,"score":55.8,"version":"GPT-5.2"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":27,"score":53.1,"version":"ERNIE 4.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":28,"score":53,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":29,"score":52.8,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":30,"score":52.7,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":31,"score":52.3,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":32,"score":52.2,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":33,"score":50.2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":34,"score":48.1,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":35,"score":48,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":36,"score":40.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":37,"score":39,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":73.2,"topic":"General VQA","total_models":1613,"url":""},{"bench_models":4,"description":"SimpleVQA variant curated by DeepSeek with everyday image question answering tasks.","github":"","has_recent":0,"id":"simplevqa-ds","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SimpleVQA-DS","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":1,"score":61.34,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":57.09,"version":"Gemini 2.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":3,"score":55.8,"version":"dots.vlm1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":4,"score":52.19,"version":"Qwen2.5 VL"}],"stars":"","top_company":"bytedance","top_model":"Seed1.5-VL-Thinking","top_model_family":"Seed","top_model_id":"seed1.5-vl-thinking","top_model_version":"Seed 1.5","top_open_source":0,"top_percent":61.34,"topic":"General VQA","total_models":1613,"url":""},{"bench_models":9,"description":"Social Interaction QA benchmark evaluating social commonsense and situational reasoning.","github":"","has_recent":0,"id":"siqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Social Interaction QA (SIQA)","paper":"","placements":[{"company":"google","family":"Gemma","model":"gemma-3-27b","place":1,"score":54.9,"version":"Gemma 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":2,"score":53.8,"version":"Motif 2"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":3,"score":53.4,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":4,"score":52.3,"version":"Llama 1"},{"company":"google","family":"Chinchilla","model":"chinchilla-70b","place":5,"score":51.3,"version":"Chinchilla"},{"company":"google","family":"Gopher","model":"gopher-280b","place":6,"score":50.6,"version":"Gopher"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":7,"score":50.4,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":8,"score":50.4,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":9,"score":48.9,"version":"Llama 1"}],"stars":"","top_company":"google","top_model":"Gemma 3 27B","top_model_family":"Gemma","top_model_id":"gemma-3-27b","top_model_version":"Gemma 3","top_open_source":1,"top_percent":54.9,"topic":"Social commonsense QA","total_models":1613,"url":""},{"bench_models":11,"description":"Semantically-Labeled Knowledge-Enhanced medical visual question answering benchmark.","github":"","has_recent":0,"id":"slake","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SLAKE","paper":"","placements":[{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":1,"score":81.6,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":2,"score":81.6,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":81.3,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":80,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":5,"score":79.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":78.7,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":76.9,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":76.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":73.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":10,"score":70.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":11,"score":54.7,"version":"Qwen3 VL"}],"stars":"","top_company":"moonshotai","top_model":"Kimi-K2.5","top_model_family":"Kimi","top_model_id":"kimi-k2.5","top_model_version":"Kimi K2.5","top_open_source":1,"top_percent":81.6,"topic":"Medical VQA","total_models":1613,"url":""},{"bench_models":5,"description":"Small molecule instruction-following and understanding benchmark.","github":"","has_recent":0,"id":"smolinstruct","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SmolInstruct","paper":"","placements":[{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":1,"score":74.8,"version":"Intern-S1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":58.3,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":53.5,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":48.2,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":36.6,"version":"Qwen3 VL"}],"stars":"","top_company":"internlm","top_model":"Intern-S1-Pro","top_model_family":"Intern","top_model_id":"intern-s1-pro","top_model_version":"Intern-S1","top_open_source":1,"top_percent":74.8,"topic":"Small molecule understanding","total_models":1613,"url":""},{"bench_models":6,"description":"Social interaction commonsense QA.","github":"","has_recent":0,"id":"socialiqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SocialIQA","paper":"","placements":[{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":1,"score":54.9,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":2,"score":51.8,"version":"Gemma 2"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":3,"score":50.87,"version":"MobileLLM P1"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":4,"score":49.7,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":5,"score":48.9,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":6,"score":45.6,"version":"Llama 3.2"}],"stars":"","top_company":"google","top_model":"Gemma 3 PT 27B","top_model_family":"Gemma","top_model_id":"gemma-3-pt-27b","top_model_version":"Gemma 3","top_open_source":1,"top_percent":54.9,"topic":"Social commonsense","total_models":1613,"url":"https://leaderboard.allenai.org/socialiqa"},{"bench_models":15,"description":"Mental visualization benchmark.","github":"","has_recent":0,"id":"spatialviz","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SpatialViz","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":65.8,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":65.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":63.2,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":4,"score":52.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":5,"score":51.3,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":46.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":43,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":41.6,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":9,"score":38.5,"version":"Grok 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":10,"score":38.4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":11,"score":38.2,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":12,"score":35.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":13,"score":35.2,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4","place":14,"score":34.8,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-4o","place":15,"score":31.1,"version":"GPT-4o"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":65.8,"topic":"Mental visualization","total_models":1613,"url":""},{"bench_models":21,"description":"Complex text-to-SQL benchmark over cross-domain databases.","github":"","has_recent":0,"id":"spider","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Spider","paper":"","placements":[{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":1,"score":82.49,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":2,"score":81.79,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":3,"score":81.37,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":4,"score":81.04,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":5,"score":80.58,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":6,"score":79.18,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":7,"score":77.55,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":8,"score":76.76,"version":"LLaDA 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":9,"score":76.43,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":10,"score":75.78,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":11,"score":72.8,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":12,"score":67.1,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3-70b","place":13,"score":67.1,"version":"Llama 3"},{"company":"mistral","family":"Codestral","model":"codestral-22b","place":14,"score":63.5,"version":"Codestral"},{"company":"deepseek","family":"DeepSeek Math","model":"deepseek-math-7b","place":15,"score":61.2,"version":"DeepSeek Math"},{"company":"deepseek","family":"DeepSeek Coder","model":"deepseek-coder-33b","place":16,"score":60,"version":"DeepSeek Coder"},{"company":"mistral","family":"Codestral","model":"codestral-mamba","place":17,"score":58.8,"version":"Codestral"},{"company":"codellama","family":"CodeLlama-34B","model":"codellama-34b","place":18,"score":50.8,"version":"CodeLlama-34B"},{"company":"google","family":"CodeGemma","model":"codegemma-1.1-7b","place":19,"score":46.3,"version":"CodeGemma"},{"company":"meta","family":"CodeLlama","model":"codellama-70b","place":20,"score":37,"version":"CodeLlama"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":21,"score":29.3,"version":"CodeLlama-7B"}],"stars":"","top_company":"antgroup","top_model":"LLaDA2.0 Flash","top_model_family":"LLaDA","top_model_id":"llada2.0-flash","top_model_version":"LLaDA 2.0","top_open_source":1,"top_percent":82.49,"topic":"Text-to-SQL","total_models":1613,"url":"https://yale-lily.github.io/spider"},{"bench_models":10,"description":"A LLM-judged benchmark measuring sycophancy and delusion reinforcement.","github":"https://github.com/EQ-bench","has_recent":0,"id":"spiral-bench","leaderboard":"https://eqbench.com/spiral-bench.html","lower_is_better":false,"metric":"percent","name":"Spiral-Bench","paper":"https://arxiv.org/abs/2312.06281","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":87,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":2,"score":86.1,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":3,"score":81.4,"version":"GPT-OSS"},{"company":"openai","family":"o4","model":"o4-mini","place":4,"score":73.3,"version":"o4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":5,"score":73,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":6,"score":49.1,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":7,"score":48.1,"version":"Llama 4"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":8,"score":47.7,"version":"Claude 3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":9,"score":43.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":10,"score":42.3,"version":"GPT-4o"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":87,"topic":"Safety / sycophancy","total_models":1613,"url":"https://eqbench.com/spiral-bench.html"},{"bench_models":5,"description":"Extractive QA from Wikipedia articles.","github":"","has_recent":0,"id":"squad-v1.1","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SQuAD v1.1","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":1,"score":89.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":2,"score":85.6,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":3,"score":81.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":4,"score":77,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":5,"score":76.4,"version":"Llama 3"}],"stars":"566","top_company":"meta","top_model":"Llama 3.1 405B Base","top_model_family":"Llama","top_model_id":"llama-3.1-405b-base","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":89.3,"topic":"Reading comprehension","total_models":1613,"url":"https://github.com/rajpurkar/SQuAD-explorer"},{"bench_models":11,"description":"Like v1.1 with unanswerable questions.","github":"","has_recent":0,"id":"squad-v2.0","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SQuAD v2.0","paper":"","placements":[{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":1,"score":90.8,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":2,"score":90.65,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":3,"score":90,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":4,"score":89.51,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":5,"score":86.5,"version":"LLaDA 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash-preview","place":6,"score":85.61,"version":"LLaDA 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":7,"score":85.21,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":8,"score":85.13,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":9,"score":84.51,"version":"LLaDA 2.1"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":10,"score":81.32,"version":"Ling 2.0"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":11,"score":75.56,"version":"Ling 2.0"}],"stars":"566","top_company":"antgroup","top_model":"LLaDA2.1-Flash (Q Mode)","top_model_family":"LLaDA","top_model_id":"llada2.1-flash-q","top_model_version":"LLaDA 2.1","top_open_source":1,"top_percent":90.8,"topic":"Reading comprehension","total_models":1613,"url":"https://github.com/rajpurkar/SQuAD-explorer"},{"bench_models":2,"description":"Streaming video understanding benchmark.","github":"","has_recent":0,"id":"streamingbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"StreamingBench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":84.4,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":2,"score":68,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":84.4,"topic":"Streaming video","total_models":1613,"url":""},{"bench_models":15,"description":"SUN RGB-D benchmark for indoor scene understanding from RGB-D imagery.","github":"","has_recent":0,"id":"sunrgbd","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SUNRGBD","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":1,"score":45.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":2,"score":42.2,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":3,"score":41,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":38.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":5,"score":38.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":6,"score":37,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":7,"score":36.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":8,"score":36.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":9,"score":35.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":10,"score":34.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":11,"score":34.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":12,"score":34.7,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":13,"score":33.8,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":14,"score":33.4,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":15,"score":32.5,"version":"Seed 1.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 Mini Minimal","top_model_family":"GPT","top_model_id":"gpt-5-mini-minimal","top_model_version":"GPT-5","top_open_source":0,"top_percent":45.8,"topic":"3D scene understanding","total_models":1613,"url":""},{"bench_models":9,"description":"Chemistry reasoning benchmark evaluating text-based chemical knowledge and problem solving.","github":"","has_recent":0,"id":"superchem","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SuperChem","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":63.2,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":2,"score":58,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":3,"score":54.4,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":4,"score":51.6,"version":"Seed 2.0"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":5,"score":48,"version":"Seed 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":43.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":7,"score":34.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":32.4,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":9,"score":16.2,"version":"Seed 2.0"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":63.2,"topic":"Chemistry reasoning","total_models":1613,"url":""},{"bench_models":82,"description":"Harder GPQA variant assessing advanced graduate-level reasoning.","github":"","has_recent":0,"id":"supergpqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SuperGPQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":75.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":74,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":73.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":72.7,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":70.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":6,"score":70.4,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":69.2,"version":"Kimi K2.5"},{"company":"bytedance","family":"Seed","model":"seed2.0-pro","place":8,"score":68.7,"version":"Seed 2.0"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":9,"score":67.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":10,"score":67.9,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed2.0-lite","place":11,"score":67.5,"version":"Seed 2.0"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":12,"score":67.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":13,"score":67.1,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":14,"score":66.8,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":15,"score":66.4,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":16,"score":66.1,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":17,"score":65.6,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":18,"score":65.5,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":19,"score":64.9,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":20,"score":64.9,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":21,"score":64.9,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":22,"score":64.8,"version":"Seed 1.8"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":23,"score":64.6,"version":"Qwen3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":24,"score":64.2,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":25,"score":63.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":26,"score":62.6,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed2.0-mini","place":27,"score":61.6,"version":"Seed 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":28,"score":61.37,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":29,"score":60.8,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":30,"score":60.59,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-mini-high","place":31,"score":60.5,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":32,"score":60.34,"version":"Ling 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":33,"score":59.8,"version":"DeepSeek V3.1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":34,"score":58.9,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":35,"score":58.8,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":36,"score":58.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":37,"score":57.8,"version":"Gemini 2.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":38,"score":57.22,"version":"Ling 1T"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":39,"score":57.2,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":40,"score":57.2,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":41,"score":56.8,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":42,"score":56.5,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":43,"score":56.4,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":44,"score":55.7,"version":"Claude 4"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":45,"score":55,"version":"JoyAI-LLM"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":46,"score":54.6,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":47,"score":54.4,"version":"Claude 3.7"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":48,"score":54.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":49,"score":54.1,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":50,"score":53.7,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":51,"score":53.4,"version":"Qwen3"},{"company":"boss","family":"Nanbeige","model":"nanbeige4-3b-thinking-2511","place":52,"score":53.2,"version":"Nanbeige4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":53,"score":53.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":54,"score":52,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":55,"score":50.8,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":56,"score":50.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":57,"score":49.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":58,"score":46.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":59,"score":46.7,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.6","place":60,"score":45.08,"version":"Seed 1.6"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":61,"score":44.7,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":62,"score":44.5,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":63,"score":44.3,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-base","place":64,"score":44.06,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":65,"score":43.6,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":66,"score":43.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":67,"score":42.8,"version":"Qwen3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":68,"score":42.6,"version":"MiMo VL"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":69,"score":42.3,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":70,"score":41.53,"version":"DeepSeek V3"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":71,"score":41.1,"version":"MiMo V2"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":72,"score":40.85,"version":"Llama 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":73,"score":40.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":74,"score":39.78,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":75,"score":39.1,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed1.5","place":76,"score":36.81,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":77,"score":36.6,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":78,"score":36.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":79,"score":35.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":80,"score":35.72,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":81,"score":34.27,"version":"Qwen3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":82,"score":32.68,"version":"Motif 2"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":83,"score":32,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-4o","place":84,"score":31.6,"version":"GPT-4o"},{"company":"essential","family":"Rnj","model":"rnj-1","place":85,"score":30.2,"version":"Rnj 1"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":86,"score":29.5,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":87,"score":29.4,"version":"InternVL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":88,"score":29.3,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":89,"score":25.4,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":90,"score":21.6,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":91,"score":21,"version":"OLMo 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":92,"score":20.5,"version":"Llama 3.1"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":93,"score":19.7,"version":"Codestral"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":75.3,"topic":"Graduate-level QA","total_models":1613,"url":""},{"bench_models":31,"description":"Supervised software engineering benchmark across many repos and issues.","github":"","has_recent":0,"id":"swe-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Bench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-codex","place":1,"score":74.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":74.4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":74.2,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":4,"score":72.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":71.8,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":70,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":7,"score":69.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":8,"score":66.6,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":9,"score":61.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":10,"score":60.6,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":11,"score":60,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":12,"score":59,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":13,"score":58.7,"version":"Gemini 3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":14,"score":54.4,"version":"LongCat-Flash-Lite"},{"company":"xai","family":"Grok","model":"grok-4","place":15,"score":48.4,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":47,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":17,"score":42,"version":"Grok 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":18,"score":41.3,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b","place":19,"score":38.8,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":20,"score":37.6,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":21,"score":34.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-a4b","place":22,"score":34,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":23,"score":32.8,"version":"Kimi Linear"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":24,"score":32,"version":"Gemini 2.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":25,"score":30.8,"version":"MiMo V2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":26,"score":28.2,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":27,"score":25.5,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":28,"score":24.8,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":29,"score":22,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":30,"score":22,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":31,"score":21.6,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":32,"score":9.4,"version":"DeepSeek V3.2"}],"stars":"3442","top_company":"openai","top_model":"GPT-5 Codex","top_model_family":"GPT","top_model_id":"gpt-5-codex","top_model_version":"GPT-5","top_open_source":0,"top_percent":74.5,"topic":"Code repair","total_models":1613,"url":"https://github.com/princeton-nlp/SWE-bench"},{"bench_models":43,"description":"Multilingual variant of SWE-Bench for issue fixing.","github":"","has_recent":0,"id":"swe-bench-multilingual","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Bench Multilingual","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":1,"score":77.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":77.5,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-5","place":3,"score":73.3,"version":"GLM 5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":4,"score":73,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":5,"score":73,"version":"Kimi K2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":6,"score":72.5,"version":"MiniMax M2.1"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":7,"score":72,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":72,"version":"GPT-5.2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":9,"score":71.7,"version":"MiMo V2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":10,"score":71.7,"version":"MiMo V2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":71.7,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":12,"score":70.2,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":13,"score":70.2,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":14,"score":69.3,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":15,"score":68,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":16,"score":68,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":17,"score":68,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":18,"score":68,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":19,"score":67.2,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":20,"score":66.7,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":21,"score":66.7,"version":"Qwen3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":22,"score":66.2,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":23,"score":65,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-4.7","place":24,"score":63.7,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":25,"score":63.7,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-next","place":26,"score":62.8,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":27,"score":62.3,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":28,"score":61.1,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":29,"score":57.9,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":30,"score":57.8,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":31,"score":56.9,"version":"Claude 4"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":32,"score":56.5,"version":"MiniMax M2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":33,"score":55.9,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":34,"score":55.9,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":35,"score":55.9,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":36,"score":55.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":37,"score":55.3,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":38,"score":54.7,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":39,"score":54.5,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":40,"score":54.5,"version":"DeepSeek V3.1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":41,"score":53.8,"version":"GLM 4.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":42,"score":53.3,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.5","place":43,"score":52.7,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":44,"score":51,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":45,"score":47.3,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":46,"score":47.3,"version":"Kimi K2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":47,"score":38.1,"version":"LongCat-Flash-Lite"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":48,"score":37.2,"version":"Kimi Linear"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":49,"score":31.5,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":50,"score":31.3,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":51,"score":25.8,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":52,"score":20.9,"version":"Qwen3"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5 Thinking","top_model_family":"Claude","top_model_id":"claude-opus-4.5-thinking","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":77.5,"topic":"Code repair (multilingual)","total_models":1613,"url":""},{"bench_models":3,"description":"SWE-Bench results using the OpenHands autonomous coding agent.","github":"","has_recent":0,"id":"swe-bench-openhands","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Bench (OpenHands)","paper":"","placements":[{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":1,"score":38.8,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":2,"score":34,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":3,"score":22,"version":"Qwen3"}],"stars":"3442","top_company":"nvidia","top_model":"NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","top_model_family":"Nemotron","top_model_id":"nvidia-nemotron-3-nano-30b-a3b-bf16","top_model_version":"Nemotron 3","top_open_source":1,"top_percent":38.8,"topic":"Code repair","total_models":1613,"url":"https://github.com/princeton-nlp/SWE-bench"},{"bench_models":16,"description":"Full SWE-Bench Pro benchmark for software-engineering agents.","github":"","has_recent":0,"id":"swe-bench-pro","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Bench Pro","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":56.9,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":2,"score":55.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":3,"score":55.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":55.6,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":5,"score":55.4,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":6,"score":55.4,"version":"MiniMax M2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":7,"score":55.4,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":54.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":52,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1-thinking","place":10,"score":50.8,"version":"GPT-5.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":11,"score":50.7,"version":"Kimi K2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":12,"score":49.7,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":13,"score":46.7,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":44.3,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-next","place":15,"score":44.3,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":43.3,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":17,"score":40.9,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":18,"score":40.6,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":19,"score":39.8,"version":"Kimi K2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":20,"score":34.6,"version":"MiniMax M2.1"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":56.9,"topic":"Software engineering","total_models":1613,"url":"https://www.swebench.com/"},{"bench_models":12,"description":"Public subset of the SWE-Bench Pro benchmark for software-engineering agents.","github":"","has_recent":0,"id":"swe-bench-pro-public","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Bench Pro (Public)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":1,"score":56.8,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":2,"score":56.8,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":3,"score":56.4,"version":"GPT-5.2 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":4,"score":56.4,"version":"GPT-5.2 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":5,"score":55.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":55.6,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":7,"score":54.2,"version":"Gemini 3.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":43.3,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5","place":9,"score":23.26,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":10,"score":22.71,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":11,"score":17.65,"version":"Claude 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro-preview-06-05","place":12,"score":13.54,"version":"Gemini 2.5"},{"company":"swe","family":"SWE Smith","model":"swesmith-32b","place":13,"score":6.84,"version":"SWE Smith"},{"company":"openai","family":"GPT","model":"gpt-4o","place":14,"score":4.92,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":15,"score":3.42,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-5.3 Codex","top_model_family":"GPT","top_model_id":"gpt-5.3-codex","top_model_version":"GPT-5.3 Codex","top_open_source":0,"top_percent":56.8,"topic":"Software engineering","total_models":1613,"url":""},{"bench_models":150,"description":"Verified subset of SWE-Bench for issue fixing.","github":"","has_recent":0,"id":"swe-bench-verified","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Bench Verified","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":80.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":2,"score":80.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":80.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":4,"score":80.8,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":5,"score":80.8,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":6,"score":80.6,"version":"Gemini 3.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":7,"score":80.2,"version":"MiniMax M2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":8,"score":80,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":9,"score":80,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":10,"score":80,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":11,"score":80,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":12,"score":80,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":13,"score":80,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":14,"score":79.6,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":15,"score":78.2,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":16,"score":78,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":17,"score":78,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":18,"score":78,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1-codex-max","place":19,"score":77.9,"version":"GPT-5.1 Codex"},{"company":"zai","family":"GLM","model":"glm-5","place":20,"score":77.8,"version":"GLM 5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":21,"score":77.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":22,"score":77.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":23,"score":77.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":24,"score":77.2,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":25,"score":76.8,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":26,"score":76.8,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":27,"score":76.4,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":28,"score":76.3,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":29,"score":76.3,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":30,"score":76.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":31,"score":76.2,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":32,"score":76,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":33,"score":75.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5","place":34,"score":74.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":35,"score":74.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":36,"score":74.9,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":37,"score":74.8,"version":"MiniMax M2.1"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":38,"score":74.6,"version":"KAT"},{"company":"openai","family":"GPT","model":"gpt-5-codex","place":39,"score":74.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":40,"score":74.5,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-medium","place":41,"score":74.4,"version":"Claude 4.5"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":42,"score":74.4,"version":"Step 3.5 Flash"},{"company":"zai","family":"GLM","model":"glm-4.7","place":43,"score":74.2,"version":"GLM 4.7"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":44,"score":74,"version":"MiniMax M2.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":45,"score":74,"version":"MiniMax M2.1"},{"company":"zai","family":"GLM","model":"glm-4.7","place":46,"score":73.8,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":47,"score":73.8,"version":"GLM 4.7"},{"company":"kuaishou","family":"KAT","model":"kat-coder","place":48,"score":73.4,"version":"KAT"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":49,"score":73.4,"version":"MiMo V2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":50,"score":73.4,"version":"MiMo V2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":51,"score":73.3,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":52,"score":73.2,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":53,"score":73.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":54,"score":73.1,"version":"DeepSeek V3.2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":55,"score":72.9,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5","place":56,"score":72.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":57,"score":72.7,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":58,"score":72.7,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":59,"score":72.5,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":60,"score":72.5,"version":"Claude 4"},{"company":"antgroup","family":"Ring","model":"ring-1t-2.5","place":61,"score":72.4,"version":"Ring 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":62,"score":72.4,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":63,"score":72,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":64,"score":72,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-high","place":65,"score":71.8,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":66,"score":71.3,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":67,"score":70.8,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-moatless-tools","place":68,"score":70.8,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-next","place":69,"score":70.6,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":70,"score":70.2,"version":"DeepSeek V3.2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":71,"score":70,"version":"LongCat-Flash"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":72,"score":69.7,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":73,"score":69.6,"version":"Qwen3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":74,"score":69.4,"version":"MiniMax M2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":75,"score":69.2,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":76,"score":69.2,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":77,"score":69.2,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":78,"score":69.2,"version":"Qwen3.5"},{"company":"openai","family":"o3","model":"o3","place":79,"score":69.1,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5.2-medium","place":80,"score":69,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":81,"score":68.4,"version":"DeepSeek V3.1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":82,"score":68,"version":"GLM 4.6"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":83,"score":67.8,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":84,"score":67.2,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":85,"score":66.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-r2e-gym","place":86,"score":66.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":87,"score":66.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-swe-agent","place":88,"score":66.6,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":89,"score":66,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":90,"score":66,"version":"DeepSeek V3.1"},{"company":"openai","family":"GPT","model":"gpt-5.1-medium","place":91,"score":66,"version":"GPT-5.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":92,"score":65.8,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":93,"score":65.8,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":94,"score":65.7,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-r2e-gym","place":95,"score":65.7,"version":"GPT-5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":96,"score":64.3,"version":"IQuest Coder V1"},{"company":"zai","family":"GLM","model":"glm-4.5","place":97,"score":64.2,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-358b","place":98,"score":64.2,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-358b-swe-agent","place":99,"score":64.2,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":100,"score":63.8,"version":"Gemini 2.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":101,"score":62.5,"version":"IQuest Coder V1"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":102,"score":62.4,"version":"KAT"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":103,"score":62.4,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":104,"score":62.3,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":105,"score":62,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":106,"score":60.6,"version":"Gemini 3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":107,"score":60.6,"version":"JoyAI-LLM"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":108,"score":60.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":109,"score":60.3,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":110,"score":59.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":111,"score":59.6,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":112,"score":59.2,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":113,"score":58.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":114,"score":57.9,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":115,"score":57.6,"version":"DeepSeek R1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":116,"score":56,"version":"MiniMax M1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":117,"score":55.7,"version":"Kimi K2"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":118,"score":55.6,"version":"MiniMax M1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b","place":119,"score":55.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-mini-swe-agent","place":120,"score":55.4,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":121,"score":55.1,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":122,"score":54.6,"version":"GPT-4.1"},{"company":"microsoft","family":"FrogBoss","model":"frogboss-32b-2510","place":123,"score":54.6,"version":"FrogBoss"},{"company":"zai","family":"GLM","model":"glm-4.5-358b","place":124,"score":54.2,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-358b-mini-swe-agent","place":125,"score":54.2,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":126,"score":54,"version":"Qwen3"},{"company":"cwm","family":"CWM","model":"cwm-32b","place":127,"score":53.9,"version":"CWM"},{"company":"zai","family":"GLM","model":"glm-4.7","place":128,"score":53.7,"version":"GLM 4.7"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":129,"score":53.4,"version":"Kimi K2"},{"company":"swe-mirror","family":"SWE-Mirror","model":"swe-mirror-lm-32b","place":130,"score":52.2,"version":"SWE-Mirror"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":131,"score":51.6,"version":"GLM 4.7"},{"company":"teleai","family":"TeleChat","model":"telechat3-36b-thinking","place":132,"score":51,"version":"TeleChat3"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":133,"score":50.6,"version":"Grok 4.1"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":134,"score":49.4,"version":"K-EXAONE"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":135,"score":49.2,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":136,"score":49,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":137,"score":48.9,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":138,"score":47.2,"version":"DeepSeek V3.2"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":139,"score":47,"version":"Seed 1.5"},{"company":"mistral","family":"Devstral","model":"devstral","place":140,"score":46.8,"version":"Devstral"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":141,"score":45.6,"version":"DeepSeek R1"},{"company":"microsoft","family":"FrogMini","model":"frogmini-14b-2510","place":142,"score":45.3,"version":"FrogMini"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":143,"score":44,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":144,"score":43.2,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-14b-thinking","place":145,"score":43.1,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":146,"score":42.7,"version":"Qwen2.5"},{"company":"deepswe","family":"DeepSWE","model":"deepswe-32b-preview","place":147,"score":42.2,"version":"DeepSWE"},{"company":"deepcogito","family":"Cogito","model":"cogito-671b-v2.1","place":148,"score":42,"version":"Cogito 671B v2.1"},{"company":"teleai","family":"TeleChat","model":"telechat3-105b-a4.7b-thinking","place":149,"score":42,"version":"TeleChat3"},{"company":"swe-rl","family":"SWE-RL","model":"swe-rl-70b","place":150,"score":41,"version":"SWE-RL"},{"company":"anthropic","family":"Claude","model":"claude-3-5-haiku-20240620","place":151,"score":40.6,"version":"Claude 3.5"},{"company":"swe-smith","family":"SWE-Smith","model":"swe-smith-lm-32b","place":152,"score":40.2,"version":"SWE-Smith"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":153,"score":38.8,"version":"DeepSeek V3"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b-thinking","place":154,"score":38.5,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":155,"score":38.4,"version":"Qwen2.5"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":156,"score":38.2,"version":"Seed Coder"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":157,"score":38,"version":"GPT-4.5"},{"company":"skywork","family":"Skywork","model":"skywork-swe-32b","place":158,"score":38,"version":"Skywork SWE"},{"company":"nvidia","family":"Nemotron Cascade","model":"nemotron-cascade-8b","place":159,"score":37.2,"version":"Nemotron Cascade"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":160,"score":36.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":161,"score":34.4,"version":"Qwen3"},{"company":"r2e-gym","family":"R2E-Gym","model":"r2e-gym-32b","place":162,"score":34.4,"version":"R2E-Gym"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":163,"score":34,"version":"GPT-OSS"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":164,"score":33.7,"version":"KAT"},{"company":"swe-fixer","family":"SWE-Fixer","model":"swe-fixer-72b","place":165,"score":32.8,"version":"SWE-Fixer"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":166,"score":32.5,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":167,"score":30.8,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o","place":168,"score":29.3,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4o-r2e-gym","place":169,"score":29.3,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":170,"score":28,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":171,"score":27.4,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":172,"score":26.7,"version":"DeepSeek Coder V2"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":173,"score":26,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":174,"score":25,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":175,"score":24.44,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":176,"score":23.6,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":177,"score":22,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":178,"score":21,"version":"Qwen3"},{"company":"essential","family":"Rnj","model":"rnj-1","place":179,"score":20.8,"version":"Rnj 1"},{"company":"swe-gym","family":"SWE-Gym","model":"swe-gym-32b","place":180,"score":20.6,"version":"SWE-Gym"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":181,"score":20.5,"version":"Qwen3"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":182,"score":17.7,"version":"Youtu-LLM"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":183,"score":7.2,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":184,"score":5.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":185,"score":4.5,"version":"Qwen3"},{"company":"mistral","family":"Codestral","model":"codestral-12b-instruct","place":186,"score":3.7,"version":"Codestral"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":187,"score":2.4,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":188,"score":1.8,"version":"Llama 3.1"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-moe","place":189,"score":1.1,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":190,"score":0.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":191,"score":0.5,"version":"Qwen2.5"},{"company":"allenai","family":"OLMo","model":"olmo-3-7b-sft","place":192,"score":0,"version":"OLMo 3"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":80.9,"topic":"Code repair","total_models":1613,"url":"https://www.swebench.com/"},{"bench_models":6,"description":"Software engineering development and bug fixing benchmark.","github":"","has_recent":0,"id":"swe-dev","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Dev","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":1,"score":67.1,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":2,"score":66.6,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":3,"score":64.7,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":4,"score":63.2,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":5,"score":61.9,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":6,"score":53.3,"version":"DeepSeek V3.1"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4","top_model_family":"Claude","top_model_id":"claude-sonnet-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":67.1,"topic":"Code repair","total_models":1613,"url":""},{"bench_models":8,"description":"Software engineering benchmark using real freelance-style issues.","github":"","has_recent":0,"id":"swe-lancer","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Lancer","paper":"https://arxiv.org/abs/2502.12115","placements":[{"company":"openai","family":"GPT","model":"gpt-5.1-codex-max","place":1,"score":79.9,"version":"GPT-5.1 Codex"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":2,"score":40.8,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":3,"score":39.1,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":4,"score":38.5,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":5,"score":37.3,"version":"GPT-4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":6,"score":30.5,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":24.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":8,"score":23,"version":"GPT-4.1"}],"stars":"","top_company":"openai","top_model":"GPT-5.1 Codex-Max","top_model_family":"GPT","top_model_id":"gpt-5.1-codex-max","top_model_version":"GPT-5.1 Codex","top_open_source":0,"top_percent":79.9,"topic":"Code repair (freelance tasks)","total_models":1613,"url":""},{"bench_models":6,"description":"Diamond subset of SWE-Lancer focusing on the hardest freelance-style issues.","github":"","has_recent":0,"id":"swe-lancer-diamond","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Lancer Diamond","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":1,"score":81.4,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":2,"score":76,"version":"GPT-5.2 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":74.6,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":4,"score":32.6,"version":"GPT-4.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":5,"score":23.3,"version":"GPT-4o"},{"company":"openai","family":"o3","model":"o3-mini","place":6,"score":10.8,"version":"o3"}],"stars":"","top_company":"openai","top_model":"GPT-5.3 Codex","top_model_family":"GPT","top_model_id":"gpt-5.3-codex","top_model_version":"GPT-5.3 Codex","top_open_source":0,"top_percent":81.4,"topic":"Code repair (freelance)","total_models":1613,"url":""},{"bench_models":3,"description":"Individual Contributor Diamond subset of SWE-Lancer.","github":"","has_recent":0,"id":"swe-lancer-ic-diamond","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Lancer IC Diamond","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":1,"score":81.4,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":2,"score":76,"version":"GPT-5.2 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":74.6,"version":"GPT-5.2"}],"stars":"","top_company":"openai","top_model":"GPT-5.3 Codex","top_model_family":"GPT","top_model_id":"gpt-5.3-codex","top_model_version":"GPT-5.3 Codex","top_open_source":0,"top_percent":81.4,"topic":"Code repair (freelance)","total_models":1613,"url":""},{"bench_models":7,"description":"Software engineering benchmark focused on performance-oriented fixes.","github":"","has_recent":0,"id":"swe-perf","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Perf","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":6.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":4.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":3,"score":3.6,"version":"GPT-5.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":4,"score":3.1,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":3,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":6,"score":1.4,"version":"MiniMax M2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":7,"score":0.9,"version":"DeepSeek V3.2"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":6.5,"topic":"Code repair","total_models":1613,"url":""},{"bench_models":5,"description":"Software engineering review benchmark for assessing code review quality.","github":"","has_recent":0,"id":"swe-review","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWE-Review","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":16.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":10.5,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":3,"score":8.9,"version":"MiniMax M2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":4,"score":6.4,"version":"DeepSeek V3.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":5,"score":3.4,"version":"MiniMax M2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":16.2,"topic":"Code review","total_models":1613,"url":""},{"bench_models":7,"description":"Software tool-use benchmark for code tasks.","github":"","has_recent":0,"id":"swt-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SWT-Bench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":1,"score":80.7,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":80.2,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":79.7,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":69.5,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":5,"score":69.3,"version":"MiniMax M2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":6,"score":62,"version":"DeepSeek V3.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":7,"score":32.8,"version":"MiniMax M2"}],"stars":"","top_company":"openai","top_model":"GPT-5.2 Thinking","top_model_family":"GPT","top_model_id":"gpt-5.2-thinking","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":80.7,"topic":"Code repair","total_models":1613,"url":""},{"bench_models":5,"description":"System prompt understanding and adherence benchmark.","github":"","has_recent":0,"id":"sysbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"SysBench","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-4.1","place":1,"score":74.1,"version":"GPT-4.1"},{"company":"tencent","family":"Hunyuan","model":"hunyuan-7b-instruct","place":2,"score":72.7,"version":"Hunyuan"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":3,"score":71.3,"version":"DeepSeek V3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":4,"score":69.8,"version":"ERNIE 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":5,"score":51.3,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-4.1","top_model_family":"GPT","top_model_id":"gpt-4.1","top_model_version":"GPT-4.1","top_open_source":0,"top_percent":74.1,"topic":"System prompts","total_models":1613,"url":""},{"bench_models":11,"description":"Tool-augmented agent evaluation in airline scenarios (TAU1).","github":"","has_recent":0,"id":"tau1-airline","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TAU1-Airline","paper":"","placements":[{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":1,"score":56,"version":"openPangu-R"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":2,"score":54,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":3,"score":49,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":4,"score":48,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":5,"score":46,"version":"Qwen3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":6,"score":45.3,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":44,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":8,"score":44,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":9,"score":40,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":10,"score":29,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":11,"score":17,"version":"Qwen3"}],"stars":"","top_company":"freedomintelligence","top_model":"openPangu-R-72B-2512 Slow Thinking","top_model_family":"openPangu-R","top_model_id":"openpangu-r-72b-2512-slow-thinking","top_model_version":"openPangu-R","top_open_source":1,"top_percent":56,"topic":"Agent tasks (airline)","total_models":1613,"url":""},{"bench_models":11,"description":"Tool-augmented agent evaluation in retail scenarios (TAU1).","github":"","has_recent":0,"id":"tau1-retail","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TAU1-Retail","paper":"","placements":[{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":1,"score":73,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":2,"score":71.3,"version":"Qwen3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":3,"score":70.1,"version":"openPangu-R"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":4,"score":69.6,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":5,"score":67.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":6,"score":67.8,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":7,"score":65.2,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":8,"score":60.9,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":9,"score":59.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":10,"score":52.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":11,"score":40.1,"version":"Qwen3"}],"stars":"","top_company":"freedomintelligence","top_model":"openPangu-R-72B-2512 Slow Thinking","top_model_family":"openPangu-R","top_model_id":"openpangu-r-72b-2512-slow-thinking","top_model_version":"openPangu-R","top_open_source":1,"top_percent":73,"topic":"Agent tasks (retail)","total_models":1613,"url":""},{"bench_models":45,"description":"Tool-augmented agent evaluation in airline scenarios (TAU2).","github":"","has_recent":0,"id":"tau2-airline","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TAU2-Airline","paper":"","placements":[{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":1,"score":76.5,"version":"LongCat-Flash"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":2,"score":70,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":3,"score":66,"version":"DeepSeek V3.2"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":4,"score":65.3,"version":"openPangu-R"},{"company":"openai","family":"o3","model":"o3","place":5,"score":64.8,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":6,"score":63.8,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":7,"score":63.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":8,"score":63,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":9,"score":63,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":10,"score":62.6,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":11,"score":62,"version":"MiniMax M1"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":12,"score":60.8,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":13,"score":60.5,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":14,"score":60.5,"version":"GPT-OSS"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":15,"score":60.4,"version":"K-EXAONE"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":16,"score":60,"version":"MiniMax M1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":17,"score":60,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":18,"score":59.6,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":19,"score":58.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":20,"score":58,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":21,"score":58,"version":"Qwen3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":22,"score":58,"version":"openPangu-R"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":23,"score":58,"version":"LongCat-Flash-Lite"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":24,"score":56.5,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":25,"score":56,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":26,"score":55.5,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":27,"score":54.5,"version":"GPT-4.1"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":28,"score":54,"version":"JoyAI-LLM"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":29,"score":53.5,"version":"DeepSeek R1"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":30,"score":52.8,"version":"GPT-OSS"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":31,"score":52.4,"version":"Solar Open"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":32,"score":52,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":33,"score":52,"version":"o3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":34,"score":52,"version":"EXAONE 4.0"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":35,"score":52,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":36,"score":50,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":37,"score":50,"version":"Gemini 2.5"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":38,"score":48,"version":"Nemotron 3"},{"company":"openai","family":"GPT","model":"gpt-4o","place":39,"score":45.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":40,"score":45.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":41,"score":45.5,"version":"Qwen3 Next"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":42,"score":44,"version":"Seed 1.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":43,"score":44,"version":"Kimi Linear"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":44,"score":42.5,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":45,"score":39,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":46,"score":38,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":47,"score":38,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":48,"score":35,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":49,"score":34.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":50,"score":32,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":51,"score":26.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":52,"score":24,"version":"Qwen3"}],"stars":"","top_company":"meituan","top_model":"LongCat-Flash-Thinking-2601","top_model_family":"LongCat","top_model_id":"longcat-flash-thinking-2601","top_model_version":"LongCat-Flash","top_open_source":1,"top_percent":76.5,"topic":"Agent tasks (airline)","total_models":1613,"url":""},{"bench_models":9,"description":"Aggregate tool-augmented agent evaluation across airline, retail, and telecom scenarios (TAU2).","github":"","has_recent":0,"id":"tau2-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TAU2-Bench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":91.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":87.1,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":86.7,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":85.4,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":5,"score":84.6,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":77,"version":"Kimi K2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":7,"score":68.1,"version":"DeepSeek V3.2"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":8,"score":59.78,"version":"Ling 2.5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":9,"score":34.58,"version":"Ling 1T"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":91.6,"topic":"Agent tasks","total_models":1613,"url":""},{"bench_models":54,"description":"Tool-augmented agent evaluation in retail scenarios (TAU2).","github":"","has_recent":0,"id":"tau2-retail","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TAU2-Retail","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":1,"score":91.9,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":91.9,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":3,"score":91.7,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":4,"score":90.8,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":5,"score":88.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":6,"score":88.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":7,"score":88.9,"version":"Claude 4.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":8,"score":88.6,"version":"LongCat-Flash"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":9,"score":86.8,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":10,"score":86.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":11,"score":86.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":86.2,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":13,"score":85.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":14,"score":85.3,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":15,"score":83.8,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":16,"score":83.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":17,"score":82.4,"version":"Claude 4.1"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":18,"score":82,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":19,"score":82,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":20,"score":82,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":21,"score":81.8,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":22,"score":81.8,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":23,"score":81.4,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":24,"score":81.1,"version":"GPT-5"},{"company":"openai","family":"o3","model":"o3","place":25,"score":80.2,"version":"o3"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":26,"score":78.7,"version":"openPangu-R"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":27,"score":78.6,"version":"K-EXAONE"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":28,"score":77.9,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":29,"score":76.5,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":30,"score":75,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":31,"score":74.8,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":32,"score":74.6,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3","place":33,"score":73.9,"version":"o3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":34,"score":73.1,"version":"LongCat-Flash-Lite"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":35,"score":71.9,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":36,"score":71.9,"version":"GLM 4.5"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":37,"score":71.4,"version":"openPangu-R"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":38,"score":70.6,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":39,"score":69.1,"version":"DeepSeek V3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":40,"score":69.1,"version":"GPT-OSS"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":41,"score":68.4,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":42,"score":67.8,"version":"Qwen3 Next"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":43,"score":67.8,"version":"MiniMax M1"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":44,"score":67.55,"version":"JoyAI-LLM"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":45,"score":67.5,"version":"EXAONE 4.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":46,"score":67,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":47,"score":66.7,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":48,"score":64.3,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":49,"score":63.9,"version":"DeepSeek R1"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":50,"score":63.5,"version":"MiniMax M1"},{"company":"openai","family":"GPT","model":"gpt-4o","place":51,"score":63.4,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":52,"score":62.28,"version":"GLM 4.7"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":53,"score":59.3,"version":"Solar Open"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":54,"score":58.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":55,"score":58.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":56,"score":57.3,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":57,"score":57,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":58,"score":57,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":59,"score":56.9,"version":"Nemotron 3"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":60,"score":55.7,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":61,"score":53.51,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":62,"score":49.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":63,"score":48.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":64,"score":38,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":65,"score":37.5,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":66,"score":18.86,"version":"Kimi Linear"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.6","top_model_family":"Claude","top_model_id":"claude-opus-4.6","top_model_version":"Claude 4.6","top_open_source":0,"top_percent":91.9,"topic":"Agent tasks (retail)","total_models":1613,"url":""},{"bench_models":91,"description":"Tool-augmented agent evaluation in telecom scenarios (TAU2).","github":"","has_recent":0,"id":"tau2-telecom","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TAU2-Telecom","paper":"","placements":[{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":1,"score":99.3,"version":"LongCat-Flash"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":99.3,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":3,"score":99.3,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":4,"score":99.3,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":5,"score":98.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":6,"score":98.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":98.7,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":98.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":9,"score":98.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":98.2,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":11,"score":98,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":98,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":98,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":14,"score":98,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":98,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":16,"score":97.9,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5","place":17,"score":96.7,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":18,"score":96.2,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5","place":19,"score":95.8,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":20,"score":93,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":21,"score":91,"version":"DeepSeek V3.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":22,"score":90,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7-flash","place":23,"score":88.6,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5-codex-high","place":24,"score":87,"version":"GPT-5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":25,"score":87,"version":"MiniMax M2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":26,"score":87,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":27,"score":87,"version":"Gemini 3"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":28,"score":87,"version":"HyperCLOVA X"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":29,"score":87,"version":"Mi:dm K 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":30,"score":86.8,"version":"MiniMax M2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":31,"score":85.8,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":32,"score":85,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":33,"score":85,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":34,"score":84.8,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":35,"score":84.2,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":36,"score":83,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3","place":37,"score":80.7,"version":"o3"},{"company":"jd","family":"JoyAI-LLM","model":"joyai-llm-flash","place":38,"score":79.83,"version":"JoyAI-LLM"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":39,"score":78.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":40,"score":78,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":41,"score":78,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":42,"score":75.7,"version":"Grok Code"},{"company":"xai","family":"Grok","model":"grok-4","place":43,"score":75,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4","place":44,"score":74.9,"version":"Grok 4"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":45,"score":73.5,"version":"K-EXAONE"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":46,"score":73.4,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":47,"score":73.4,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":48,"score":73,"version":"Kimi K2"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":49,"score":72.8,"version":"LongCat-Flash-Lite"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":50,"score":71.5,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":51,"score":71.4,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":52,"score":71,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.6","place":53,"score":71,"version":"GLM 4.6"},{"company":"zai","family":"GLM","model":"glm-4.6","place":54,"score":70.5,"version":"GLM 4.6"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":55,"score":68.4,"version":"Apriel 1.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":56,"score":68,"version":"Apriel 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":57,"score":67,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":58,"score":66,"version":"GPT-OSS"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":59,"score":66,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":60,"score":65.8,"version":"GPT-OSS"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":61,"score":65.8,"version":"Grok 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":62,"score":65.8,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":63,"score":65.8,"version":"GPT-OSS"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":64,"score":65.8,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":65,"score":64.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":66,"score":64.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":67,"score":60.3,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":68,"score":60.2,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":69,"score":60.2,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":70,"score":60,"version":"GPT-OSS"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":71,"score":57.8,"version":"Apriel 1.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":72,"score":57.7,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":73,"score":57,"version":"Claude 4"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":74,"score":55.6,"version":"Solar Open"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":75,"score":55,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":76,"score":54.7,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":77,"score":54.1,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":78,"score":54,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":79,"score":53.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-2507","place":80,"score":53.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-2507","place":81,"score":53,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":82,"score":52,"version":"Magistral 1.2"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":83,"score":50.2,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":84,"score":49.7,"version":"GPT-OSS"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":85,"score":49.6,"version":"Claude 4"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-slow-thinking","place":86,"score":49.4,"version":"openPangu-R"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":87,"score":49,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini","place":88,"score":48.9,"version":"GPT-4.1"},{"company":"freedomintelligence","family":"openPangu-R","model":"openpangu-r-72b-2512-fast-thinking","place":89,"score":48.8,"version":"openPangu-R"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":90,"score":47.4,"version":"GPT-OSS"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":91,"score":47.3,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":92,"score":46.5,"version":"GLM 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":93,"score":46,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":94,"score":45.6,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":95,"score":45.6,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":96,"score":45.2,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":97,"score":43.9,"version":"Qwen3 Next"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":98,"score":42.2,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":99,"score":41.5,"version":"Qwen3 Next"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":100,"score":38.6,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":101,"score":37.1,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":102,"score":37,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":103,"score":36.5,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":104,"score":34,"version":"GPT-4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":105,"score":34,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":106,"score":33.9,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":107,"score":32.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":108,"score":32.5,"version":"DeepSeek V3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":109,"score":31.6,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":110,"score":31.6,"version":"Gemini 2.5"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":111,"score":31.5,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":112,"score":29.8,"version":"Qwen3"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":113,"score":28.1,"version":"Llama Nemotron v1.5"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":114,"score":28.1,"version":"GLM 4.5"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":115,"score":28,"version":"Llama Nemotron v1.5"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":116,"score":27.8,"version":"Magistral 1.2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":117,"score":27.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":118,"score":27.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":119,"score":26.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":120,"score":25.4,"version":"Qwen3"},{"company":"tii","family":"Falcon","model":"falcon-h1r-7b","place":121,"score":25.4,"version":"Falcon H1R"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":122,"score":24.6,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":123,"score":24.6,"version":"GLM 4.5"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":124,"score":23.7,"version":"EXAONE 4.0"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":125,"score":22.1,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":126,"score":21.93,"version":"Gemini 2.5"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":127,"score":18,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":128,"score":17.8,"version":"Llama 4"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":129,"score":17.3,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":130,"score":17,"version":"EXAONE 4.0"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":131,"score":16.9,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":132,"score":15.68,"version":"Kimi Linear"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":133,"score":13.2,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":134,"score":12.3,"version":"Qwen3"},{"company":"nvidia","family":"Nemotron-H","model":"nemotron-h-47b-reasoning","place":135,"score":11.4,"version":"Nemotron-H"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":136,"score":4.39,"version":"Qwen3"}],"stars":"","top_company":"meituan","top_model":"LongCat-Flash-Thinking-2601","top_model_family":"LongCat","top_model_id":"longcat-flash-thinking-2601","top_model_version":"LongCat-Flash","top_open_source":1,"top_percent":99.3,"topic":"Agent tasks (telecom)","total_models":1613,"url":""},{"bench_models":6,"description":"Temporal reasoning benchmark evaluating understanding of time-related concepts in videos and images.","github":"","has_recent":0,"id":"tempcompass","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TempCompass","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":88,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":86.9,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":83.9,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":83.7,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":82.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":75.8,"version":"Gemini 2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":7,"score":75.1,"version":"ERNIE 4.5 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":88,"topic":"Temporal reasoning","total_models":1613,"url":""},{"bench_models":79,"description":"Command-line task completion benchmark for agents.","github":"https://github.com/laude-institute/terminal-bench","has_recent":0,"id":"terminal-bench","leaderboard":"https://www.tbench.ai/leaderboard","lower_is_better":false,"metric":"percent","name":"Terminal-Bench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":1,"score":61.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":59.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":3,"score":59.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-codex","place":4,"score":58.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":54.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":6,"score":54.2,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":7,"score":54,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":53.4,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":9,"score":52.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":10,"score":51,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":11,"score":50.8,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":12,"score":50,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":13,"score":50,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":14,"score":49,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5","place":15,"score":48.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":16,"score":47.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":17,"score":47.6,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":18,"score":47.6,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":19,"score":47.1,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":20,"score":46.5,"version":"Claude 4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":21,"score":46.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":22,"score":46.4,"version":"DeepSeek V3.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":23,"score":46.3,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-thinking","place":24,"score":45,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":25,"score":44.5,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":26,"score":44.5,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":27,"score":44.5,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":28,"score":43.8,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":29,"score":43.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":30,"score":43.3,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":31,"score":43.2,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":32,"score":42.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":33,"score":42.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":34,"score":42.1,"version":"GPT-5.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":35,"score":41.3,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":36,"score":41.3,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":37,"score":41.3,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":38,"score":41,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":39,"score":41,"version":"GLM 4.7"},{"company":"zai","family":"GLM","model":"glm-4.6","place":40,"score":40.5,"version":"GLM 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":41,"score":40.1,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":42,"score":39.9,"version":"GLM 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":43,"score":39.2,"version":"Claude 4"},{"company":"xai","family":"Grok","model":"grok-4","place":44,"score":38.8,"version":"Grok 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":45,"score":37.7,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":46,"score":37.7,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0711","place":47,"score":37.5,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":48,"score":37.5,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":49,"score":37.5,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":50,"score":36.7,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":51,"score":36.4,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":52,"score":36.4,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-max-instruct-preview","place":53,"score":36.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":54,"score":36.3,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":55,"score":35.5,"version":"Claude 4"},{"company":"glaive","family":"Glaive","model":"glaive-swe-v1","place":56,"score":35.3,"version":"Glaive SWE"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":57,"score":35.2,"version":"Claude 3.7"},{"company":"meituan","family":"LongCat","model":"longcat-flash-lite","place":58,"score":33.75,"version":"LongCat-Flash-Lite"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":59,"score":33.7,"version":"Kimi K2"},{"company":"xai","family":"Grok","model":"grok-4","place":60,"score":33,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":61,"score":33,"version":"Grok 4.1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":62,"score":33,"version":"IQuest Coder V1"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":63,"score":33,"version":"IQuest Coder V1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":64,"score":32.6,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":65,"score":31.8,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":66,"score":31.3,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":67,"score":31.3,"version":"DeepSeek V3.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":68,"score":31.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":69,"score":30.3,"version":"GPT-4.1"},{"company":"openai","family":"o3","model":"o3","place":70,"score":30.2,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":71,"score":30,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":72,"score":30,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":73,"score":29.8,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":74,"score":29.5,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":75,"score":27.8,"version":"Kimi K2"},{"company":"openai","family":"o4","model":"o4-mini","place":76,"score":27.5,"version":"o4"},{"company":"xai","family":"Grok","model":"grok-4-fast-thinking","place":77,"score":26.3,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":78,"score":26.3,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":79,"score":26.3,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":80,"score":26.1,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":81,"score":25.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":82,"score":23.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":83,"score":23.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":84,"score":23.1,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":85,"score":22.5,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":86,"score":22.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":87,"score":21.3,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-code-fast","place":88,"score":20,"version":"Grok Code"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-instruct","place":89,"score":20,"version":"Kimi Linear"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":90,"score":20,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":91,"score":18.8,"version":"Grok 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":92,"score":17.5,"version":"Llama 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":93,"score":17.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":94,"score":15.19,"version":"Qwen3 Next"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":95,"score":13.8,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":96,"score":13.5,"version":"Qwen3"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":97,"score":10.1,"version":"KAT"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":98,"score":9.1,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":99,"score":8.3,"version":"GPT-4.1"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":100,"score":7.9,"version":"KAT"},{"company":"openai","family":"GPT","model":"gpt-4o","place":101,"score":7.8,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":102,"score":4.5,"version":"Qwen2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":103,"score":3.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":104,"score":3.4,"version":"Qwen3"},{"company":"bytedance","family":"Seed","model":"seed-coder-8b-instruct","place":105,"score":2.5,"version":"Seed Coder"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":106,"score":2.3,"version":"Kimi Dev"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-coder-v2-lite-instruct","place":107,"score":0,"version":"DeepSeek Coder V2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-7b-instruct","place":108,"score":0,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-14b-instruct","place":109,"score":0,"version":"Qwen2.5"}],"stars":"637","top_company":"anthropic","top_model":"Claude Sonnet 4.5 (Thinking)","top_model_family":"Claude","top_model_id":"claude-4.5-sonnet-thinking","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":61.3,"topic":"Agent terminal tasks","total_models":1613,"url":"https://www.tbench.ai/"},{"bench_models":64,"description":"Second-generation Terminal-Bench leaderboard for end-to-end terminal agents.","github":"https://github.com/laude-institute/terminal-bench","has_recent":0,"id":"terminal-bench-2.0","leaderboard":"https://www.tbench.ai/leaderboard","lower_is_better":false,"metric":"percent","name":"Terminal-Bench 2.0","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-loop-instruct","place":1,"score":81.4,"version":"IQuest Coder V1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":80.9,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash-preview","place":3,"score":78,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":4,"score":77.3,"version":"GPT-5.3 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":5,"score":77.3,"version":"GPT-5.3 Codex"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":77.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":7,"score":76.3,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro-preview","place":8,"score":76.2,"version":"Gemini 3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":9,"score":75.2,"version":"IQuest Coder V1"},{"company":"kuaishou","family":"KAT","model":"kat-dev-72b-exp","place":10,"score":74.6,"version":"KAT"},{"company":"zai","family":"GLM","model":"glm-4.7","place":11,"score":73.8,"version":"GLM 4.7"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":12,"score":73.1,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":13,"score":71.3,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct-0905","place":14,"score":69.2,"version":"Kimi K2"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":15,"score":67,"version":"Qwen3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":16,"score":65.4,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":17,"score":65.4,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":18,"score":64.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":19,"score":64.7,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":20,"score":64,"version":"GPT-5.2 Codex"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":21,"score":64,"version":"GPT-5.2 Codex"},{"company":"kuaishou","family":"KAT","model":"kat-dev-32b","place":22,"score":62.4,"version":"KAT"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":23,"score":62.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":24,"score":62.2,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-dev-72b","place":25,"score":60.4,"version":"Kimi Dev"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":26,"score":59.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":27,"score":59.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":28,"score":59.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":29,"score":59.1,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.1-codex-max","place":30,"score":58.1,"version":"GPT-5.1 Codex"},{"company":"zai","family":"GLM","model":"glm-5","place":31,"score":56.2,"version":"GLM 5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":32,"score":56.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":33,"score":56.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":34,"score":54.2,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":35,"score":54,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":36,"score":54,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":37,"score":52.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-30b-a3b-instruct","place":38,"score":51.9,"version":"Qwen3"},{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":39,"score":51,"version":"Step 3.5 Flash"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":40,"score":51,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":41,"score":51,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":42,"score":51,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":43,"score":50.8,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":44,"score":50,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":45,"score":49.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":46,"score":49.4,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":47,"score":48.3,"version":"Kimi K2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":48,"score":47.9,"version":"MiniMax M2.1"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":49,"score":47.6,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":50,"score":47.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":51,"score":47.6,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":52,"score":46.5,"version":"Claude 4.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":53,"score":46.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":54,"score":46.4,"version":"DeepSeek V3.2"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":55,"score":45.2,"version":"Seed 1.8"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-instruct-2507","place":56,"score":45.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":57,"score":44.6,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-codex","place":58,"score":44.3,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":59,"score":42.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":60,"score":42.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":61,"score":42.8,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":62,"score":41.6,"version":"Qwen3.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":63,"score":41,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":64,"score":40.5,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":65,"score":39.3,"version":"DeepSeek V3.2"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":66,"score":38.5,"version":"MiMo V2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":67,"score":38,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.7","place":68,"score":37.1,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-next","place":69,"score":36.2,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":70,"score":35.7,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":71,"score":35.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":72,"score":32.6,"version":"Gemini 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":73,"score":32.6,"version":"MiniMax M2.1"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":74,"score":31.9,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-2.5-1t","place":75,"score":31.46,"version":"Ling 2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":76,"score":30,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":77,"score":29.8,"version":"Claude 4.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":78,"score":29.21,"version":"DeepSeek V3.2"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":79,"score":29,"version":"K-EXAONE"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":80,"score":27.8,"version":"Kimi K2"},{"company":"xai","family":"Grok","model":"grok-4","place":81,"score":27.2,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":82,"score":25.8,"version":"Grok Code"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":83,"score":25.4,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.6","place":84,"score":24.5,"version":"GLM 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":85,"score":23.6,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":86,"score":22.5,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":87,"score":18.7,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":88,"score":18.7,"version":"GPT-OSS"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":89,"score":17.1,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":90,"score":16.9,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":91,"score":13.3,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":92,"score":11.5,"version":"GPT-5"},{"company":"antgroup","family":"Ling","model":"ling-1t","place":93,"score":8.99,"version":"Ling 1T"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":94,"score":3.4,"version":"GPT-OSS"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Loop-Instruct","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-loop-instruct","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":81.4,"topic":"Agent terminal tasks","total_models":1613,"url":"https://www.tbench.ai/"},{"bench_models":53,"description":"Hard subset of Terminal-Bench command-line agent tasks.","github":"","has_recent":0,"id":"terminal-bench-hard","leaderboard":"https://artificialanalysis.ai/evaluations/terminalbench-hard","lower_is_better":false,"metric":"percent","name":"Terminal-Bench Hard","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.1-high","place":1,"score":43,"version":"GPT-5.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":39,"version":"Gemini 3"},{"company":"xai","family":"Grok","model":"grok-4","place":3,"score":38,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4","place":4,"score":37.6,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5-codex-high","place":5,"score":36,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-codex","place":6,"score":35.5,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":7,"score":35.4,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":8,"score":35.4,"version":"DeepSeek V3.2"},{"company":"openai","family":"o3","model":"o3","place":9,"score":34.8,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":10,"score":33.3,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.7","place":11,"score":33.3,"version":"GLM 4.7"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":12,"score":33.3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":13,"score":33,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":14,"score":32.1,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":15,"score":32,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":16,"score":31,"version":"Claude 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":17,"score":30.6,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5","place":18,"score":30.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":19,"score":30.5,"version":"GPT-5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":20,"score":30.5,"version":"MiMo V2"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":21,"score":29.8,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":22,"score":29.8,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":23,"score":29.1,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":24,"score":29,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-terminus","place":25,"score":28.4,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":26,"score":26,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":27,"score":25.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":28,"score":25,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":29,"score":24.8,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":30,"score":24.1,"version":"DeepSeek V3.1"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":31,"score":24.1,"version":"MiniMax M2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":32,"score":24,"version":"MiniMax M2"},{"company":"zai","family":"GLM","model":"glm-4.6","place":33,"score":23.6,"version":"GLM 4.6"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":34,"score":23,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":35,"score":22.7,"version":"Kimi K2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905","place":36,"score":22.7,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":37,"score":22,"version":"GPT-OSS"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":38,"score":22,"version":"GPT-OSS"},{"company":"zai","family":"GLM","model":"glm-4.5","place":39,"score":21.3,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":40,"score":19.1,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":41,"score":18,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":42,"score":18,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4-fast","place":43,"score":17.7,"version":"Grok 4"},{"company":"openai","family":"GPT","model":"gpt-5-minimal","place":44,"score":17.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":45,"score":16,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":46,"score":15,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":47,"score":14.9,"version":"DeepSeek R1"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":48,"score":13,"version":"Magistral 1.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-2507","place":49,"score":13,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-medium-1.2","place":50,"score":12.8,"version":"Magistral 1.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":51,"score":12.8,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-2507","place":52,"score":12.8,"version":"Qwen3"},{"company":"naver","family":"HyperCLOVA X","model":"hyperclovax-seed-think","place":53,"score":11,"version":"HyperCLOVA X"},{"company":"openai","family":"GPT","model":"gpt-oss-20b-high","place":54,"score":10,"version":"GPT-OSS"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":55,"score":10,"version":"Apriel 1.5"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker","place":56,"score":9.9,"version":"Apriel 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b","place":57,"score":9.2,"version":"Qwen3 Next"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":58,"score":8.5,"version":"Nemotron 3"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":59,"score":6.4,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":60,"score":6,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-4-maverick","place":61,"score":6,"version":"Llama 4"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":62,"score":5.7,"version":"GPT-OSS"},{"company":"servicenow","family":"Apriel","model":"apriel-1.5-15b-thinker-self-reported","place":63,"score":5.7,"version":"Apriel 1.5"},{"company":"nvidia","family":"Llama","model":"llama-nemotron-super-49b-v1.5","place":64,"score":5,"version":"Llama Nemotron v1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":65,"score":5,"version":"Qwen3"},{"company":"mistral","family":"Magistral","model":"magistral-small-1.2","place":66,"score":4.3,"version":"Magistral 1.2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":67,"score":4,"version":"EXAONE 4.0"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b","place":68,"score":3.5,"version":"EXAONE 4.0"},{"company":"kt","family":"Mi:dm","model":"midm-k-2.5-pro","place":69,"score":2,"version":"Mi:dm K 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-2507","place":70,"score":1.4,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-5.1 High","top_model_family":"GPT","top_model_id":"gpt-5.1-high","top_model_version":"GPT-5.1","top_open_source":0,"top_percent":43,"topic":"Agent terminal tasks","total_models":1613,"url":"https://www.tbench.ai/"},{"bench_models":11,"description":"Terminal-Bench Terminus track assessing end-to-end terminal tool use.","github":"https://github.com/laude-institute/terminal-bench","has_recent":0,"id":"terminal-bench-terminus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Terminal-Bench Terminus","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3.1-pro","place":1,"score":68.5,"version":"Gemini 3.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":2,"score":65.4,"version":"Claude 4.6"},{"company":"openai","family":"GPT","model":"gpt-5.3-codex","place":3,"score":64.7,"version":"GPT-5.3 Codex"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.6","place":4,"score":59.1,"version":"Claude 4.6"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":56.9,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-codex","place":6,"score":54,"version":"GPT-5.2 Codex"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":7,"score":30.3,"version":"GPT-4.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":8,"score":25,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":9,"score":16.8,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":10,"score":16.3,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":11,"score":6.6,"version":"Qwen3"}],"stars":"","top_company":"google","top_model":"Gemini 3.1 Pro","top_model_family":"Gemini","top_model_id":"gemini-3.1-pro","top_model_version":"Gemini 3.1","top_open_source":0,"top_percent":68.5,"topic":"Agent terminal tasks","total_models":1613,"url":"https://www.tbench.ai/"},{"bench_models":18,"description":"Text-based video game benchmark.","github":"","has_recent":0,"id":"textquests","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TextQuests","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":41,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":38.7,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":3,"score":37,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":4,"score":36.4,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":5,"score":34.2,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":31,"version":"Claude 4.5"},{"company":"xai","family":"Grok","model":"grok-4","place":7,"score":27.8,"version":"Grok 4"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":23.2,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":9,"score":21.2,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":10,"score":18.3,"version":"Kimi K2"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":11,"score":17.6,"version":"GPT-5"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":12,"score":16.4,"version":"Grok 4.1"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":13,"score":15.1,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":14,"score":14.4,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":15,"score":13.1,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-max","place":16,"score":12.4,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":17,"score":11.7,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":18,"score":1.2,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":41,"topic":"Text-based video games","total_models":1613,"url":""},{"bench_models":15,"description":"Harmfulness evaluation on TextQuests scenarios.","github":"","has_recent":0,"id":"textquests-harm","leaderboard":"","lower_is_better":true,"metric":"percent","name":"TextQuests Harm","paper":"","placements":[{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":1,"score":9.1,"version":"Grok 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":2,"score":11.7,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":15.9,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":4,"score":16.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":5,"score":16.7,"version":"GPT-5.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":17.1,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":7,"score":17.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":17.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":9,"score":18.2,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":20.3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":11,"score":20.5,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":12,"score":21.4,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":13,"score":21.6,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":14,"score":22.8,"version":"Gemini 2.5"},{"company":"xai","family":"Grok","model":"grok-4","place":15,"score":24.3,"version":"Grok 4"}],"stars":"","top_company":"xai","top_model":"Grok 4.1 Fast","top_model_family":"Grok","top_model_id":"grok-4.1-fast","top_model_version":"Grok 4.1","top_open_source":0,"top_percent":9.1,"topic":"Harmful propensities","total_models":1613,"url":""},{"bench_models":79,"description":"Visual question answering that requires reading text in images.","github":"","has_recent":0,"id":"textvqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TextVQA","paper":"","placements":[{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":1,"score":90.7,"version":"Ovis 2.6"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":90.6,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":3,"score":88,"version":"GLM 4.6V"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":4,"score":87.2,"version":"GPT-5"},{"company":"unknown","family":"PLM","model":"plm-8b","place":5,"score":86.5,"version":"PLM"},{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":6,"score":85.7,"version":"Molmo2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":7,"score":85.5,"version":"Qwen2 VL"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":8,"score":85,"version":"Molmo2"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":9,"score":84.7,"version":"Molmo2"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":10,"score":84.4,"version":"InternVL 2"},{"company":"apple","family":"Manzano","model":"manzano-30b","place":11,"score":84.4,"version":"Manzano"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":12,"score":84.3,"version":"Qwen2 VL"},{"company":"unknown","family":"PLM","model":"plm-3b","place":13,"score":84.3,"version":"PLM"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":14,"score":83.7,"version":"Eagle 2.5"},{"company":"jina","family":"jina-VLM","model":"jina-vlm-2.4b-20251117","place":15,"score":83.2,"version":"jina-VLM"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":16,"score":83.1,"version":"Molmo"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":17,"score":82.8,"version":"Qwen3 VL"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-26b","place":18,"score":82.4,"version":"InternVL 2.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":19,"score":82.3,"version":"Llama 3.2"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":20,"score":82.2,"version":"MiniCPM V 4.5"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":21,"score":81.7,"version":"Molmo"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":22,"score":81.5,"version":"Keye-VL 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":23,"score":81,"version":"Qwen3 VL"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":24,"score":80.5,"version":"LLaVA OneVision"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":25,"score":80.4,"version":"Molmo"},{"company":"apple","family":"Manzano","model":"manzano-3b","place":26,"score":80.1,"version":"Manzano"},{"company":"jondurbin","family":"Bagel","model":"bagel-14b","place":27,"score":80,"version":"Bagel"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":28,"score":79.7,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-2b","place":29,"score":79.7,"version":"Qwen2 VL"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":30,"score":79.6,"version":"GLM 4.1V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":31,"score":79.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5vl-3b","place":32,"score":79.3,"version":"Qwen2.5 VL"},{"company":"apple","family":"MM1","model":"mm1.5-30b","place":33,"score":79.2,"version":"MM1.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":34,"score":79.1,"version":"GPT-5"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":35,"score":78.8,"version":"MolmoE"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":36,"score":78.7,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":37,"score":78.7,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":38,"score":78.7,"version":"GPT-5"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":39,"score":78.3,"version":"LLaVA OneVision"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":40,"score":78.2,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-4v","place":41,"score":78,"version":"GPT-4V"},{"company":"salesforce","family":"BLIP","model":"blip-3o-4b","place":42,"score":78,"version":"BLIP-3O"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":43,"score":77.9,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":44,"score":77.4,"version":"GPT-4o"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":45,"score":77.4,"version":"InternVL 2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-7b","place":46,"score":77.4,"version":"Qwen2.5 Omni"},{"company":"opengvlab","family":"Intern","model":"internvl3-2b","place":47,"score":77,"version":"InternVL 3"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-4b","place":48,"score":76.8,"version":"InternVL 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":49,"score":76.8,"version":"Gemini 2.5"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":50,"score":76.7,"version":"Cambrian 1"},{"company":"nyu","family":"Cambrian","model":"cambrian-34b","place":51,"score":76.7,"version":"Cambrian"},{"company":"apple","family":"MM1","model":"mm1.5-3b","place":52,"score":76.5,"version":"MM1.5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-2b","place":53,"score":76.5,"version":"InternVL 3.5"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":54,"score":75.7,"version":"Pixtral"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":55,"score":74.9,"version":"FastVLM"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-2b","place":56,"score":74.3,"version":"InternVL 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":57,"score":74.1,"version":"Claude 3.5"},{"company":"bytedance","family":"MiniCPM","model":"minicpm-v-2.0-3b","place":58,"score":74.1,"version":"MiniCPM V 2.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":59,"score":74.1,"version":"Gemini 3"},{"company":"apple","family":"MM1","model":"mm1-30b","place":60,"score":73.5,"version":"MM1"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":61,"score":73,"version":"Gemini 2.5"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":62,"score":72,"version":"Phi 3.5"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":63,"score":71.7,"version":"Cambrian 1"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":64,"score":71,"version":"xGen-MM"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":65,"score":70.4,"version":"FastVLM"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":66,"score":70.3,"version":"Gemini 2.5"},{"company":"microsoft","family":"Phi","model":"phi-3-vision-4b","place":67,"score":70.1,"version":"Phi-3"},{"company":"bytedance","family":"LLaVA","model":"llava-next-34b","place":68,"score":69.5,"version":"LLaVA-NeXT"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":69,"score":67.5,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":70,"score":67.3,"version":"Claude 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":71,"score":67.2,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":72,"score":67,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-nano-2","place":73,"score":65.9,"version":"Gemini Nano"},{"company":"baai","family":"Emu","model":"emu3-8b","place":74,"score":64.7,"version":"Emu3"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":75,"score":64.5,"version":"FastVLM"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":76,"score":61.3,"version":"LLaVA 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":77,"score":58.2,"version":"LLaVA 1.5"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":78,"score":56,"version":"PaliGemma"},{"company":"bytedance","family":"Janus","model":"janus-pro-7b","place":79,"score":45.6,"version":"Janus Pro"},{"company":"bytedance","family":"Janus","model":"janus-pro-1.5b","place":80,"score":41.9,"version":"Janus Pro"},{"company":"nvidia","family":"VILA","model":"vila1.5-3b","place":81,"score":0,"version":"VILA 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":82,"score":0,"version":"GPT-4o"}],"stars":"","top_company":"alibaba","top_model":"Ovis2.6-30B-A3B","top_model_family":"Ovis","top_model_id":"ovis2.6-30b-a3b","top_model_version":"Ovis 2.6","top_open_source":1,"top_percent":90.7,"topic":"Text-based VQA","total_models":1613,"url":"https://textvqa.org/"},{"bench_models":16,"description":"TIIF-Bench long prompt score for text-to-image generation.","github":"","has_recent":0,"id":"tiif-bench-long","leaderboard":"","lower_is_better":false,"metric":"score","name":"TIIF-Bench Long","paper":"","placements":[{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":1,"score":88.52,"version":"Seedream"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":2,"score":88.29,"version":"GPT Image 1"},{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":3,"score":88.26,"version":"Nano Banana"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":4,"score":88.08,"version":"Seedream"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":5,"score":86.83,"version":"Qwen-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":6,"score":84.93,"version":"Qwen-Image"},{"company":"zai","family":"Z-Image","model":"z-image","place":7,"score":83.01,"version":"Z-Image"},{"company":"zai","family":"GLM","model":"glm-image","place":8,"score":81.02,"version":"GLM-Image"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":9,"score":80.05,"version":"Z-Image"},{"company":"black-forest-labs","family":"FLUX.1","model":"flux-1-dev","place":10,"score":71.78,"version":"FLUX.1"},{"company":"jondurbin","family":"Bagel","model":"bagel","place":11,"score":71.7,"version":"Bagel"},{"company":"openai","family":"DALL-E","model":"dall-e-3","place":12,"score":70.81,"version":"DALL-E 3"},{"company":"stabilityai","family":"Stable Diffusion 3","model":"sd3-medium","place":13,"score":66.09,"version":"SD3"},{"company":"bytedance","family":"Janus","model":"janus-pro","place":14,"score":65.01,"version":"Janus Pro"},{"company":"unknown","family":"Show-o","model":"show-o2","place":15,"score":58.86,"version":"Show-o2"},{"company":"stabilityai","family":"Stable Diffusion","model":"sd-xl","place":16,"score":42.13,"version":"SD XL"}],"stars":"","top_company":"bytedance","top_model":"Seedream 4.5","top_model_family":"Seedream","top_model_id":"seedream-4.5","top_model_version":"Seedream","top_open_source":0,"top_percent":88.52,"topic":"Text-to-image","total_models":1613,"url":""},{"bench_models":16,"description":"TIIF-Bench short prompt score for text-to-image generation.","github":"","has_recent":0,"id":"tiif-bench-short","leaderboard":"","lower_is_better":false,"metric":"score","name":"TIIF-Bench Short","paper":"","placements":[{"company":"unknown","family":"Nano Banana","model":"nano-banana-2.0","place":1,"score":91,"version":"Nano Banana"},{"company":"bytedance","family":"Seedream","model":"seedream-4.5","place":2,"score":90.49,"version":"Seedream"},{"company":"bytedance","family":"Seedream","model":"seedream-4.0","place":3,"score":90.45,"version":"Seedream"},{"company":"openai","family":"GPT Image","model":"gpt-image-1-high","place":4,"score":89.15,"version":"GPT Image 1"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image","place":5,"score":86.14,"version":"Qwen-Image"},{"company":"qwen","family":"Qwen-Image","model":"qwen-image-2512","place":6,"score":83.24,"version":"Qwen-Image"},{"company":"zai","family":"GLM","model":"glm-image","place":7,"score":81.01,"version":"GLM-Image"},{"company":"zai","family":"Z-Image","model":"z-image","place":8,"score":80.2,"version":"Z-Image"},{"company":"zai","family":"Z-Image","model":"z-image-turbo","place":9,"score":77.73,"version":"Z-Image"},{"company":"openai","family":"DALL-E","model":"dall-e-3","place":10,"score":74.96,"version":"DALL-E 3"},{"company":"jondurbin","family":"Bagel","model":"bagel","place":11,"score":71.5,"version":"Bagel"},{"company":"black-forest-labs","family":"FLUX.1","model":"flux-1-dev","place":12,"score":71.09,"version":"FLUX.1"},{"company":"stabilityai","family":"Stable Diffusion 3","model":"sd3-medium","place":13,"score":67.46,"version":"SD3"},{"company":"bytedance","family":"Janus","model":"janus-pro","place":14,"score":66.5,"version":"Janus Pro"},{"company":"unknown","family":"Show-o","model":"show-o2","place":15,"score":59.72,"version":"Show-o2"},{"company":"stabilityai","family":"Stable Diffusion","model":"sd-xl","place":16,"score":54.96,"version":"SD XL"}],"stars":"","top_company":"unknown","top_model":"Nano Banana 2.0","top_model_family":"Nano Banana","top_model_id":"nano-banana-2.0","top_model_version":"Nano Banana","top_open_source":0,"top_percent":91,"topic":"Text-to-image","total_models":1613,"url":""},{"bench_models":6,"description":"Benchmark for tool-integrated reasoning with visual models.","github":"","has_recent":0,"id":"tir-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TIR-Bench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":1,"score":59.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":2,"score":55.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":53.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":4,"score":29.8,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":27.6,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":6,"score":24.6,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-27B","top_model_family":"Qwen","top_model_id":"qwen3.5-27b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":59.8,"topic":"Tool-integrated reasoning","total_models":1613,"url":""},{"bench_models":2,"description":"Long-form summarization benchmark with nine-domain TLDR prompts plus extended variations.","github":"","has_recent":0,"id":"tldr9-plus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TLDR9+","paper":"","placements":[{"company":"meta","family":"MobileLLM","model":"mobilellm-p1","place":1,"score":16.8,"version":"MobileLLM"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":2,"score":16.8,"version":"Llama 3.2"}],"stars":"","top_company":"meta","top_model":"MobileLLM P1","top_model_family":"MobileLLM","top_model_id":"mobilellm-p1","top_model_version":"MobileLLM","top_open_source":1,"top_percent":16.8,"topic":"Summarization","total_models":1613,"url":""},{"bench_models":4,"description":"Temporal ordering and motion analysis benchmark (TOMATO).","github":"","has_recent":0,"id":"tomato","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TOMATO","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":60.8,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":55.8,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":50.3,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":44.9,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":60.8,"topic":"Temporal understanding","total_models":1613,"url":""},{"bench_models":11,"description":"Composite tool-use suite measuring multi-domain tool invocation success (Pass@1).","github":"","has_recent":0,"id":"tool-decathlon","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Tool-Decathlon","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":43.8,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":43.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":3,"score":38.6,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":38.3,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":5,"score":36.4,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":6,"score":35.2,"version":"DeepSeek V3.2"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":7,"score":29,"version":"GPT-5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":8,"score":27.8,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":9,"score":18.8,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":10,"score":17.6,"version":"Kimi K2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":11,"score":16,"version":"MiniMax M2"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":43.8,"topic":"Agent tool-use","total_models":1613,"url":""},{"bench_models":12,"description":"Long-horizon, real-world software tool-use tasks.","github":"","has_recent":0,"id":"toolathlon","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Toolathlon","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-flash","place":1,"score":49.4,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":2,"score":46.3,"version":"GPT-5.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":3,"score":43.5,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":4,"score":43.5,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":5,"score":41.7,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":6,"score":38.9,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":38.9,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":36.4,"version":"Gemini 3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":9,"score":35.2,"version":"DeepSeek V3.2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":10,"score":16.7,"version":"MiniMax M2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":11,"score":10.5,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":12,"score":3.7,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Flash","top_model_family":"Gemini","top_model_id":"gemini-3-flash","top_model_version":"Gemini 3","top_open_source":0,"top_percent":49.4,"topic":"Agentic software tasks","total_models":1613,"url":""},{"bench_models":7,"description":"Evaluates hierarchical/tree-structured reasoning and planning capabilities in LLMs/VLMs.","github":"","has_recent":0,"id":"treebench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TreeBench","paper":"https://arxiv.org/abs/2507.07999","placements":[{"company":"zai","family":"GLM","model":"glm-4.6v","place":1,"score":51.4,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":2,"score":50.1,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":3,"score":45.7,"version":"GLM 4.6V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":4,"score":41.5,"version":"Kimi VL"},{"company":"stepfun","family":"Step","model":"step-3","place":5,"score":41.3,"version":"Step 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":6,"score":37.5,"version":"GLM 4.1V"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":7,"score":33.8,"version":"Gemma 3"}],"stars":"","top_company":"zai","top_model":"GLM-4.6V","top_model_family":"GLM","top_model_id":"glm-4.6v","top_model_version":"GLM 4.6V","top_open_source":1,"top_percent":51.4,"topic":"Reasoning with tree structures","total_models":1613,"url":""},{"bench_models":4,"description":"Triadic question answering benchmark evaluating world knowledge and reasoning.","github":"","has_recent":0,"id":"triqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TriQA","paper":"","placements":[{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":1,"score":82.2,"version":"Mixtral"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":2,"score":78.4,"version":"Mixtral"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":3,"score":77.57,"version":"Llama 2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":4,"score":68.8,"version":"Mistral"}],"stars":"","top_company":"mistral","top_model":"Mixtral 8x22B","top_model_family":"Mixtral","top_model_id":"mixtral-8x22b","top_model_version":"Mixtral","top_open_source":1,"top_percent":82.2,"topic":"Knowledge QA","total_models":1613,"url":""},{"bench_models":49,"description":"Open-domain question answering benchmark built from trivia and web evidence.","github":"https://github.com/mandarjoshi90/triviaqa","has_recent":0,"id":"triviaqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TriviaQA","paper":"https://arxiv.org/abs/1705.03551","placements":[{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":1,"score":85.5,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":2,"score":85.5,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":3,"score":85.1,"version":"Kimi K2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":4,"score":83.9,"version":"DeepSeek V3.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":5,"score":83.5,"version":"DeepSeek V3.1"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":6,"score":83.3,"version":"Trinity Large"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-base","place":7,"score":82.9,"version":"DeepSeek V3"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":8,"score":80.96,"version":"Trinity Large"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":9,"score":80.3,"version":"MiMo V2"},{"company":"zai","family":"GLM","model":"glm-4.5-base","place":10,"score":80,"version":"GLM 4.5"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":11,"score":78.2,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":12,"score":75.2,"version":"Kimi Linear"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":13,"score":73,"version":"Llama 2"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":14,"score":72.93,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":15,"score":72.55,"version":"LLaDA 2.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":16,"score":72.2,"version":"Motif 2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":17,"score":71.5,"version":"Mixtral"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":18,"score":69.9,"version":"Mistral"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":19,"score":69.76,"version":"Ling 2.0"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":20,"score":69.6,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":21,"score":68.5,"version":"Llama 1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":22,"score":66.88,"version":"LLaDA 2.0"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":23,"score":66.2,"version":"Moonlight"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":24,"score":65.61,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":25,"score":64,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":26,"score":63.8,"version":"Llama 2"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":27,"score":63.4,"version":"Gemma 2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":28,"score":62.5,"version":"Mistral"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":29,"score":56.6,"version":"Llama 2"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":30,"score":55.63,"version":"Ling 2.0"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":31,"score":54.24,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":32,"score":53.33,"version":"LLaDA 2.1"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":33,"score":53.2,"version":"Gemma 2"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":34,"score":52.51,"version":"Qwen3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":35,"score":51.33,"version":"LLaDA 2.0"},{"company":"arcee","family":"AFM","model":"afm-4.5b","place":36,"score":47.66,"version":"AFM"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":37,"score":44.32,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-4b-it","place":38,"score":42.1,"version":"Gemma 3"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":39,"score":39.85,"version":"MobileLLM P1"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":40,"score":39.8,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":41,"score":36.7,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm-3-3b","place":42,"score":35.1,"version":"SmolLM3"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":43,"score":34.9,"version":"CodeLlama-7B"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":44,"score":28.1,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":45,"score":23.81,"version":"Llama 3.2"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":46,"score":22.5,"version":"SmolLM1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":47,"score":20.9,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":48,"score":15.4,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":49,"score":4.3,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":50,"score":4.3,"version":"SmolLM2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":51,"score":0.757,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":52,"score":0.582,"version":"Llama TFree HAT"}],"stars":"","top_company":"google","top_model":"Gemma 3 PT 27B","top_model_family":"Gemma","top_model_id":"gemma-3-pt-27b","top_model_version":"Gemma 3","top_open_source":1,"top_percent":85.5,"topic":"Open-domain QA","total_models":1613,"url":"https://nlp.cs.washington.edu/triviaqa/"},{"bench_models":5,"description":"TriviaQA subset answering using Wikipedia evidence.","github":"","has_recent":0,"id":"triviaqa-wiki","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TriviaQA-Wiki","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":1,"score":91.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":2,"score":91.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":3,"score":89.8,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3-70b-base","place":4,"score":89.7,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3-8b-base","place":5,"score":78.5,"version":"Llama 3"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":6,"score":77.6,"version":"Llama 3.1"}],"stars":"","top_company":"meta","top_model":"Llama 3.1 405B Base","top_model_family":"Llama","top_model_id":"llama-3.1-405b-base","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":91.8,"topic":"Open-domain QA","total_models":1613,"url":"http://nlp.cs.washington.edu/triviaqa/"},{"bench_models":4,"description":"TrustLLM benchmark for trustworthiness and safety behaviors.","github":"","has_recent":0,"id":"trustllm","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TrustLLM","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":1,"score":88.4,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":2,"score":86.4,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":3,"score":70.2,"version":"Qwen2.5"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":4,"score":65,"version":"IQuest Coder V1"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Coder-480B-A35B-Instruct","top_model_family":"Qwen","top_model_id":"qwen3-coder-480b-a35b-instruct","top_model_version":"Qwen3","top_open_source":1,"top_percent":88.4,"topic":"Safety / reliability","total_models":1613,"url":""},{"bench_models":89,"description":"Measures whether a model imitates human falsehoods (truthfulness).","github":"","has_recent":0,"id":"truthfulqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TruthfulQA","paper":"https://arxiv.org/abs/2109.07958","placements":[{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":1,"score":71.43,"version":"Solar Mini"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":2,"score":70.28,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":3,"score":69.9,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":4,"score":69.8,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3","place":5,"score":68.7,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":6,"score":68.26,"version":"Mistral v0.2"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-dpo","place":7,"score":67.9,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b","place":8,"score":67.6,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":9,"score":66.95,"version":"Qwen1.5"},{"company":"IBM","family":"Granite","model":"granite-3.2-8b-instruct","place":10,"score":66.92,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.3-8b-instruct","place":11,"score":66.86,"version":"Granite 3.3"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-instruct","place":12,"score":66.8,"version":"Llama 3.1"},{"company":"nousresearch","family":"Hermes","model":"hermes-3-llama-3.1-70b","place":13,"score":66.5,"version":"Hermes 3"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":14,"score":66.15,"version":"Llama 3.3"},{"company":"IBM","family":"Granite","model":"granite-3.1-8b-instruct","place":15,"score":65.84,"version":"Granite 3.1"},{"company":"tii","family":"Falcon","model":"falcon-h1-34b-instruct","place":16,"score":65.27,"version":"Falcon H1"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":17,"score":64.26,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b-instruct","place":18,"score":63.1,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b","place":19,"score":62.74,"version":"Llama 4"},{"company":"nvidia","family":"Nemotron","model":"llama-3.1-nemotron-70b","place":20,"score":62.6,"version":"Llama 3.1 Nemotron"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":21,"score":62.41,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-9b-it","place":22,"score":61.4,"version":"Gemma 2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":23,"score":61.02,"version":"Gemma 3"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":24,"score":60.5,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":25,"score":60.5,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b-chat","place":26,"score":60.36,"version":"Qwen1.5"},{"company":"tii","family":"Falcon","model":"falcon-h1-7b","place":27,"score":59.91,"version":"Falcon H1"},{"company":"IBM","family":"Granite","model":"granite-3.2-2b-instruct","place":28,"score":59.8,"version":"Granite 3.2"},{"company":"IBM","family":"Granite","model":"granite-3.1-2b-instruct","place":29,"score":59.79,"version":"Granite 3.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.1-instruct","place":30,"score":59.35,"version":"Bielik v2.1"},{"company":"IBM","family":"Granite","model":"granite-3.3-2b-instruct","place":31,"score":58.97,"version":"Granite 3.3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":32,"score":58.58,"version":"Qwen3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.2-instruct","place":33,"score":58.34,"version":"Bielik v2.2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":34,"score":58.11,"version":"Qwen2.5"},{"company":"IBM","family":"Granite","model":"granite-4.0-tiny-preview","place":35,"score":58.07,"version":"Granite 4.0"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":36,"score":57.71,"version":"Phi 3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.3-instruct","place":37,"score":57.42,"version":"Bielik v2.3"},{"company":"allenai","family":"Magpie","model":"magpie-8b","place":38,"score":57,"version":"Magpie"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.1","place":39,"score":56.28,"version":"Mistral v0.1"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-dpo","place":40,"score":56.1,"version":"Tulu 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-70b-sft","place":41,"score":55.7,"version":"Tulu 3"},{"company":"tii","family":"Falcon","model":"falcon3-7b","place":42,"score":55.58,"version":"Falcon 3"},{"company":"mistral","family":"Ministral","model":"ministral-8b-instruct","place":43,"score":55.5,"version":"Ministral"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-instruct","place":44,"score":55.1,"version":"Llama 3.1"},{"company":"tii","family":"Falcon","model":"falcon3-10b","place":45,"score":55.05,"version":"Falcon 3"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b","place":46,"score":55,"version":"Tulu 3"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5-0106-gemma","place":47,"score":54.93,"version":"OpenChat 3.5 0106"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":48,"score":54.9,"version":"OLMo 3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.0-instruct","place":49,"score":54.65,"version":"Bielik v2.0"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0-instruct","place":50,"score":54.25,"version":"Bielik v3.0"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":51,"score":54,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-7b-chat","place":52,"score":53.54,"version":"Qwen1.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.6-instruct","place":53,"score":53.43,"version":"Bielik v2.6"},{"company":"tii","family":"Falcon","model":"falcon-h1-3b","place":54,"score":53.19,"version":"Falcon H1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":55,"score":53.19,"version":"Qwen3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.5-instruct","place":56,"score":53.17,"version":"Bielik v2.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":57,"score":52.99,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b","place":58,"score":52.06,"version":"Qwen1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":59,"score":51.75,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":60,"score":51.68,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-0.8b","place":61,"score":51.65,"version":"Qwen3"},{"company":"meta","family":"Llama","model":"meta-llama-3-8b-instruct","place":62,"score":51.65,"version":"Llama 3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3-instruct","place":63,"score":50.79,"version":"Bielik v3"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b-deep","place":64,"score":50.48,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3.2-3b-base","place":65,"score":50.27,"version":"Llama 3.2"},{"company":"tii","family":"Falcon","model":"falcon3-3b","place":66,"score":50.06,"version":"Falcon 3"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":67,"score":49.7,"version":"Llama 3.1"},{"company":"qwen","family":"Qwen","model":"qwen-14b","place":68,"score":49.43,"version":"Qwen"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":69,"score":49.39,"version":"Qwen3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0","place":70,"score":47.65,"version":"Bielik v3.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-llama-8b","place":71,"score":47.43,"version":"DeepSeek R1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-v0.1","place":72,"score":47.16,"version":"Bielik v0.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-7b","place":73,"score":47.14,"version":"DeepSeek R1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":74,"score":47.05,"version":"Qwen2.5"},{"company":"allenai","family":"Llama","model":"llama-3.1-tulu-3-8b-sft","place":75,"score":46.8,"version":"Tulu 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-0.5b","place":76,"score":46.19,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-instruct-v0.1","place":77,"score":46.18,"version":"Bielik v0.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2","place":78,"score":46.13,"version":"Bielik v2"},{"company":"tii","family":"Falcon","model":"falcon-h1-1.5b","place":79,"score":45.98,"version":"Falcon H1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3","place":80,"score":45.63,"version":"Bielik v3"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":81,"score":45.6,"version":"Llama 3.0"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":82,"score":44.91,"version":"gemma-7b"},{"company":"tii","family":"Falcon","model":"falcon-h1-0.5b-instruct","place":83,"score":44.12,"version":"Falcon H1"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":84,"score":42.93,"version":"Llama 3"},{"company":"tii","family":"Falcon","model":"falcon3-1b","place":85,"score":42.31,"version":"Falcon 3"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":86,"score":42.17,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.1","place":87,"score":42.15,"version":"Mistral v0.1"},{"company":"meta","family":"Llama","model":"llama-3.2-1.24b-base","place":88,"score":42.08,"version":"Llama 3.2"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.2","place":89,"score":41.76,"version":"Mistral v0.2"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":90,"score":0.191,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":91,"score":0.176,"version":"Llama TFree HAT"}],"stars":"","top_company":"upstage","top_model":"SOLAR-10.7B-Instruct-v1.0","top_model_family":"Solar","top_model_id":"solar-10.7b-instruct-v1.0","top_model_version":"Solar Mini","top_open_source":1,"top_percent":71.43,"topic":"Truthfulness / hallucination","total_models":1613,"url":"https://github.com/sylinrl/TruthfulQA"},{"bench_models":2,"description":"German translation of the TruthfulQA benchmark.","github":"","has_recent":0,"id":"truthfulqa-de","leaderboard":"","lower_is_better":false,"metric":"norm_prob_mass","name":"TruthfulQA (DE)","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":1,"score":0.174,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":2,"score":0.17,"version":"Llama TFree HAT"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B Instruct","top_model_family":"Llama","top_model_id":"llama-3.3-70b-instruct","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":0.174,"topic":"Truthfulness / hallucination (German)","total_models":1613,"url":""},{"bench_models":4,"description":"Benchmark for TV show video comprehension and QA.","github":"","has_recent":0,"id":"tvbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TVBench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":71.5,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":71.1,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":67.4,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":66.6,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":71.5,"topic":"TV comprehension","total_models":1613,"url":""},{"bench_models":6,"description":"Typologically diverse QA across languages.","github":"","has_recent":0,"id":"tydiqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"TydiQA","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":1,"score":34.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-base","place":2,"score":31.7,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-maverick-base","place":3,"score":31.7,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-base","place":4,"score":31.5,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-base","place":5,"score":31.5,"version":"Llama 4"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":6,"score":29.9,"version":"Llama 3.1"}],"stars":"313","top_company":"meta","top_model":"Llama 3.1 405B Base","top_model_family":"Llama","top_model_id":"llama-3.1-405b-base","top_model_version":"Llama 3.1","top_open_source":1,"top_percent":34.3,"topic":"Cross-lingual QA","total_models":1613,"url":"https://github.com/google-research-datasets/tydiqa"},{"bench_models":5,"description":"Benchmark focusing on generated code artifacts quality.","github":"","has_recent":0,"id":"u-artifacts","leaderboard":"","lower_is_better":false,"metric":"percent","name":"U-Artifacts","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":57.8,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":56.8,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":49.2,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":37.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":33.4,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":57.8,"topic":"Agentic coding artifacts","total_models":1613,"url":""},{"bench_models":27,"description":"V* benchmark accuracy.","github":"","has_recent":0,"id":"v-star","leaderboard":"","lower_is_better":false,"metric":"percent","name":"V*","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":95.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":2,"score":93.7,"version":"Qwen3.5"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":3,"score":93.2,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":93.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":5,"score":92.7,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":88,"version":"Gemini 3"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":7,"score":87.4,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":8,"score":86.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":9,"score":85.9,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":10,"score":85.3,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":11,"score":81.7,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":12,"score":81.2,"version":"Qwen3 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":13,"score":80.6,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":14,"score":80.1,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":15,"score":79,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":16,"score":77,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":17,"score":75.9,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":18,"score":74.9,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":19,"score":74.3,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":20,"score":73.9,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":21,"score":73.8,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":22,"score":72.8,"version":"InternVL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":23,"score":71.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":24,"score":69.1,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":25,"score":67,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":26,"score":58.6,"version":"Claude 4.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":27,"score":50.8,"version":"Gemma 3"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":28,"score":37.7,"version":"Claude 4"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":95.8,"topic":"Multimodal reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Visual commonsense reasoning benchmark.","github":"","has_recent":0,"id":"vcrbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VCRBench","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":59.8,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":53.4,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":3,"score":51.8,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":51.4,"version":"Gemini 3"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":59.8,"topic":"Visual commonsense reasoning","total_models":1613,"url":""},{"bench_models":26,"description":"Virology Capabilities Test: a benchmark that measures an LLM's ability to troubleshoot complex virology laboratory protocols.","github":"","has_recent":0,"id":"vct","leaderboard":"https://www.virologytest.ai/","lower_is_better":false,"metric":"percent","name":"VCT","paper":"https://arxiv.org/abs/2504.16137","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":100,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":2,"score":100,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":3,"score":100,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":4,"score":100,"version":"DeepSeek V3.2"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":5,"score":100,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":6,"score":100,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-haiku","place":7,"score":100,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":8,"score":99.5,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":9,"score":97.6,"version":"GPT-5.1"},{"company":"openai","family":"GPT","model":"gpt-5-nano","place":10,"score":94.6,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":11,"score":80.7,"version":"GPT-5.2"},{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":12,"score":61,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":13,"score":56.1,"version":"Grok 4.1"},{"company":"xai","family":"Grok","model":"grok-4","place":14,"score":55,"version":"Grok 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":15,"score":49.7,"version":"Claude 4.5"},{"company":"openai","family":"o3","model":"o3","place":16,"score":43.8,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":17,"score":42.9,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":18,"score":41.1,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":19,"score":39.7,"version":"Claude 4.5"},{"company":"openai","family":"o4","model":"o4-mini","place":20,"score":37,"version":"o4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":21,"score":36,"version":"Claude 4"},{"company":"openai","family":"o1","model":"o1","place":22,"score":35.4,"version":"o1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":23,"score":31.5,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":24,"score":30.8,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-4.5","place":25,"score":28.3,"version":"GPT-4.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":26,"score":18.8,"version":"GPT-4o"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":100,"topic":"Virology capability (protocol troubleshooting)","total_models":1613,"url":""},{"bench_models":13,"description":"Long-horizon agentic task benchmark evaluating sustained goal completion.","github":"","has_recent":0,"id":"vending-bench-2","leaderboard":"","lower_is_better":false,"metric":"net_worth_mean","name":"Vending-Bench 2","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":5478.16,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":5478,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":4967,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-5","place":4,"score":4432,"version":"GLM 5"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":5,"score":3952,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":6,"score":3839,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":7,"score":3838.74,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":8,"score":3635,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":9,"score":3591,"version":"GPT-5.2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":10,"score":2377,"version":"GLM 4.7"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":11,"score":1473.43,"version":"GPT-5.1"},{"company":"xai","family":"Grok","model":"grok-4.1-fast","place":12,"score":1107,"version":"Grok 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":13,"score":574,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":14,"score":573.64,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":15,"score":549,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":5478.16,"topic":"Long-horizon agentic tasks","total_models":1613,"url":""},{"bench_models":5,"description":"Vibe evaluation on Android tasks.","github":"","has_recent":0,"id":"vibe-android","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Vibe Android","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":92.2,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":2,"score":89.7,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":87.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":78.7,"version":"Gemini 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":5,"score":69.2,"version":"MiniMax M2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":92.2,"topic":"Vibe evaluation (Android)","total_models":1613,"url":""},{"bench_models":7,"description":"Aggregate Vibe evaluation score.","github":"","has_recent":0,"id":"vibe-average","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Vibe Average","paper":"","placements":[{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":1,"score":88.6,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":85.2,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":82.4,"version":"Gemini 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":4,"score":67.5,"version":"MiniMax M2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.6","place":5,"score":55.6,"version":"Claude 4.6"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":6,"score":55.2,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.5","place":7,"score":54.2,"version":"MiniMax M2.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":8,"score":42.4,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":36.9,"version":"Gemini 3"}],"stars":"","top_company":"minimax","top_model":"MiniMax M2.1","top_model_family":"MiniMax","top_model_id":"minimax-m2.1","top_model_version":"MiniMax M2.1","top_open_source":1,"top_percent":88.6,"topic":"Vibe evaluation","total_models":1613,"url":""},{"bench_models":5,"description":"Vibe evaluation on backend tasks.","github":"","has_recent":0,"id":"vibe-backend","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Vibe Backend","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":98,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":90.8,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":3,"score":86.7,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":78.7,"version":"Gemini 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":5,"score":67.3,"version":"MiniMax M2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":98,"topic":"Vibe evaluation (backend)","total_models":1613,"url":""},{"bench_models":5,"description":"Vibe evaluation on iOS tasks.","github":"","has_recent":0,"id":"vibe-ios","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Vibe iOS","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":90,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":2,"score":88,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":81.2,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":75.8,"version":"Gemini 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":5,"score":39.5,"version":"MiniMax M2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":90,"topic":"Vibe evaluation (iOS)","total_models":1613,"url":""},{"bench_models":5,"description":"Vibe evaluation on simulation tasks.","github":"","has_recent":0,"id":"vibe-simulation","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Vibe Simulation","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":89.2,"version":"Gemini 3"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":2,"score":87.1,"version":"MiniMax M2.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":84,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":79.1,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":5,"score":77,"version":"MiniMax M2"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":89.2,"topic":"Vibe evaluation (simulation)","total_models":1613,"url":""},{"bench_models":5,"description":"Vibe evaluation on web tasks.","github":"","has_recent":0,"id":"vibe-web","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Vibe Web","paper":"","placements":[{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":1,"score":91.5,"version":"MiniMax M2.1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":89.5,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":3,"score":89.1,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":87.3,"version":"Claude 4.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":5,"score":80.4,"version":"MiniMax M2"}],"stars":"","top_company":"minimax","top_model":"MiniMax M2.1","top_model_family":"MiniMax","top_model_id":"minimax-m2.1","top_model_version":"MiniMax M2.1","top_open_source":1,"top_percent":91.5,"topic":"Vibe evaluation (web)","total_models":1613,"url":""},{"bench_models":11,"description":"VLM aesthetic evaluation with GPT scores.","github":"","has_recent":0,"id":"vibeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VibeEval","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":76.39,"version":"Gemini 2.5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":2,"score":69.24,"version":"dots.vlm1"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":68.59,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":4,"score":64.7,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":5,"score":60.13,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":6,"score":54.7,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":7,"score":47.7,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":8,"score":47.2,"version":"MiMo VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":9,"score":44,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":10,"score":43.6,"version":"InternVL"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":11,"score":39,"version":"Claude 3.7"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":76.39,"topic":"Aesthetic/visual quality","total_models":1613,"url":""},{"bench_models":12,"description":"Multimodal evaluation of video understanding and reasoning.","github":"","has_recent":0,"id":"video-mme","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Video-MME","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":1,"score":76.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":2,"score":74.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":3,"score":73.3,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":4,"score":72,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":5,"score":71.6,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":6,"score":71.6,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":7,"score":71.1,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":8,"score":70,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":9,"score":61.9,"version":"Qwen3 VL"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-2.2b","place":10,"score":52.1,"version":"SmolVLM2"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-0.5b","place":11,"score":42.2,"version":"SmolVLM2"},{"company":"huggingface","family":"SmolVLM","model":"smolvlm2-0.256b","place":12,"score":33.7,"version":"SmolVLM2"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL 32B Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-32b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":76.6,"topic":"Video understanding (multimodal)","total_models":1613,"url":"https://video-mme.github.io/home_page.html"},{"bench_models":4,"description":"Video question answering benchmark focused on detective-style clues.","github":"","has_recent":0,"id":"videoholmes","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VideoHolmes","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":65.5,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":64.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":62.4,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":59.1,"version":"Seed 1.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":65.5,"topic":"Video QA","total_models":1613,"url":""},{"bench_models":7,"description":"Video multimodal evaluation suite (VideoMME).","github":"","has_recent":0,"id":"videomme","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VideoMME","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":88.4,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":87.8,"version":"Seed 1.8"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":3,"score":87.4,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":86.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":5,"score":86,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":6,"score":83,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":7,"score":79,"version":"Qwen3 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":88.4,"topic":"Multimodal video evaluation","total_models":1613,"url":""},{"bench_models":32,"description":"Video understanding benchmark without subtitles.","github":"","has_recent":0,"id":"videomme-nosub","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VideoMME (w/o sub)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":87.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":2,"score":85.8,"version":"GPT-5.2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":85.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":4,"score":84.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":5,"score":83.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":6,"score":83.7,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":83.2,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":8,"score":82.8,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":9,"score":82.5,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":10,"score":81.4,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":11,"score":79,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":12,"score":79,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":13,"score":78.9,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":14,"score":77.9,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":15,"score":75.6,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":16,"score":75.3,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":17,"score":74.6,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":18,"score":74.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":19,"score":73.3,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":20,"score":71.4,"version":"Qwen3 VL"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":21,"score":71.1,"version":"Ovis 2.6"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":22,"score":69.3,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":23,"score":69.3,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":24,"score":68.2,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":25,"score":67.8,"version":"Kimi VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":26,"score":67.4,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":27,"score":66.9,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":28,"score":66.3,"version":"InternVL"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":29,"score":65.6,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":30,"score":65.1,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":31,"score":58.9,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":32,"score":52.6,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":87.7,"topic":"Video understanding","total_models":1613,"url":""},{"bench_models":15,"description":"Video understanding benchmark with subtitles.","github":"","has_recent":0,"id":"videomme-sub","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VideoMME (w/sub)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":88.4,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":2,"score":87.5,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":87.4,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":87.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":5,"score":87,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":6,"score":86.6,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":86,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":8,"score":83.8,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":9,"score":83.5,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":10,"score":81.1,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":11,"score":80.7,"version":"GLM 4.5V"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":12,"score":77.6,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":13,"score":73.6,"version":"GLM 4.1V"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":14,"score":71.9,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":15,"score":68.4,"version":"Gemma 3"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":88.4,"topic":"Video understanding","total_models":1613,"url":""},{"bench_models":50,"description":"Video-based extension of MMMU evaluating temporal multimodal reasoning and perception across disciplines.","github":"","has_recent":0,"id":"videommmu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VideoMMMU","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":87.6,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-flash","place":2,"score":86.9,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":3,"score":86.6,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":4,"score":86.6,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-extra-high","place":5,"score":85.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":6,"score":85.9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":7,"score":85.9,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":8,"score":84.7,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":9,"score":84.6,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":10,"score":84.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":84.4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":12,"score":83.6,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":13,"score":83.3,"version":"o3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":14,"score":82.7,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":15,"score":82.5,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":16,"score":82.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":17,"score":82,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":18,"score":81.4,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":19,"score":81.4,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5.1","place":20,"score":80.4,"version":"GPT-5.1"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":21,"score":80.4,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":22,"score":80,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":23,"score":80,"version":"Qwen3 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":24,"score":79.2,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":25,"score":77.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet-thinking","place":26,"score":77.8,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":27,"score":77.6,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":28,"score":76.2,"version":"Claude 4.1"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":29,"score":74.7,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":30,"score":71.9,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":31,"score":70.1,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":32,"score":68.7,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":33,"score":67.3,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":34,"score":67.3,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":35,"score":65.3,"version":"Qwen3 VL"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":36,"score":65.2,"version":"Kimi VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":37,"score":63,"version":"Gemini 2.5"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":38,"score":62.4,"version":"GLM 4.5V"},{"company":"openai","family":"GPT","model":"gpt-4o","place":39,"score":61.2,"version":"GPT-4o"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":40,"score":61,"version":"GLM 4.1V"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":41,"score":60.2,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":42,"score":56.7,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":43,"score":56.2,"version":"Qwen3 VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":44,"score":54.5,"version":"Gemma 3"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":45,"score":53.1,"version":"MiMo VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":46,"score":48.9,"version":"InternVL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":47,"score":47.4,"version":"Qwen2.5 VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":48,"score":43.3,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":49,"score":41.9,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":50,"score":40.2,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":87.6,"topic":"Multimodal video understanding","total_models":1613,"url":""},{"bench_models":4,"description":"Video reasoning benchmark assessing temporal and causal understanding.","github":"","has_recent":0,"id":"videoreasonbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VideoReasonBench","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":59.7,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":59.5,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":52.8,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":18.7,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":59.7,"topic":"Video reasoning","total_models":1613,"url":""},{"bench_models":4,"description":"Simple question answering over short videos.","github":"","has_recent":0,"id":"videosimpleqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VideoSimpleQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":71.9,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":69.6,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":67.8,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":59.2,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":71.9,"topic":"Video QA","total_models":1613,"url":""},{"bench_models":4,"description":"Video-grounded dialogue and description benchmark.","github":"","has_recent":0,"id":"vispeak","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ViSpeak","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":89,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":84.5,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":79,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed-1.5-vl","place":4,"score":77,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":89,"topic":"Video dialogue","total_models":1613,"url":""},{"bench_models":3,"description":"Visual puzzle solving benchmark evaluating reasoning and pattern recognition capabilities.","github":"","has_recent":0,"id":"visualpuzzle","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VisualPuzzle","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":57.8,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":2,"score":55.8,"version":"Gemini 2.5"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":3,"score":52.5,"version":"ERNIE 4.5 VL"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":57.8,"topic":"Visual reasoning","total_models":1613,"url":""},{"bench_models":28,"description":"Average accuracy on VisualWebBench.","github":"https://github.com/VisualWebBench/VisualWebBench","has_recent":0,"id":"visualwebbench-avg","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VisualWebBench","paper":"https://arxiv.org/abs/2404.05955","placements":[{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":1,"score":83.8,"version":"Holo1.5"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":2,"score":82.6,"version":"Holo1.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":3,"score":80.2,"version":"GPT-4o"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":4,"score":80.2,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":5,"score":80.2,"version":"MiMo VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":6,"score":79.7,"version":"UI-TARS"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":7,"score":79.3,"version":"Claude 3.7"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":8,"score":78.5,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":9,"score":76.3,"version":"Qwen2.5 VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":10,"score":74.1,"version":"UI-Venus"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":11,"score":69,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-3-sonnet","place":12,"score":65.8,"version":"Claude 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":13,"score":64.8,"version":"Gemini 1.5"},{"company":"openai","family":"GPT","model":"gpt-4v","place":14,"score":64.6,"version":"GPT-4V"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":15,"score":62.7,"version":"InternVL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":16,"score":60.9,"version":"UI-Venus"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":17,"score":58.9,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-3","place":18,"score":58.5,"version":"Claude 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":19,"score":58,"version":"Qwen2.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":20,"score":54.1,"version":"Holo1"},{"company":"microsoft","family":"LLaVA","model":"llava-1.6-34b","place":21,"score":50.5,"version":"LLaVA 1.6"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":22,"score":49.7,"version":"Gemma 3"},{"company":"google","family":"Gemini","model":"gemini-1.0-pro","place":23,"score":48,"version":"Gemini 1.0"},{"company":"microsoft","family":"LLaVA","model":"llava-1.6-13b","place":24,"score":39.4,"version":"LLaVA 1.6"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":25,"score":38.1,"version":"Holo1"},{"company":"microsoft","family":"LLaVA","model":"llava-1.6-7b","place":26,"score":36,"version":"LLaVA 1.6"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-vl-7b","place":27,"score":28.1,"version":"DeepSeek VL"},{"company":"nvidia","family":"VILA","model":"vila-13b","place":28,"score":27.3,"version":"VILA"}],"stars":"","top_company":"hcompany","top_model":"Holo1.5-72B","top_model_family":"Holo","top_model_id":"holo1.5-72b","top_model_version":"Holo1.5","top_open_source":1,"top_percent":83.8,"topic":"Web UI understanding","total_models":1613,"url":""},{"bench_models":17,"description":"Logical reasoning and compositionality benchmark for visual-language models.","github":"https://github.com/VisuLogic-Benchmark/VisuLogic-Eval","has_recent":0,"id":"visulogic","leaderboard":"https://visulogic-benchmark.github.io/VisuLogic/","lower_is_better":false,"metric":"percent","name":"VisuLogic","paper":"https://arxiv.org/abs/2504.15279","placements":[{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":1,"score":52.5,"version":"ERNIE 4.5 VL"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":2,"score":35.9,"version":"Seed 1.5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":3,"score":35,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":4,"score":34.4,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":5,"score":32.2,"version":"dots.vlm1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":31.6,"version":"Gemini 2.5"},{"company":"opengvlab","family":"Intern","model":"internvl2.5-38b-rl","place":7,"score":31.1,"version":"InternVL 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":29.8,"version":"Gemini 2.5"},{"company":"openai","family":"o3","model":"o3","place":9,"score":29.5,"version":"o3"},{"company":"openai","family":"GPT","model":"gpt-5","place":10,"score":28.5,"version":"GPT-5"},{"company":"skywork","family":"Skywork","model":"skywork-r1v3-38b","place":11,"score":28.5,"version":"Skywork R1V3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":12,"score":28.5,"version":"GPT-5"},{"company":"bytedance","family":"Doubao","model":"doubao-1.5-vision-pro","place":13,"score":28.1,"version":"Doubao 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro","place":14,"score":28,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b-instruct-rl","place":15,"score":28,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":16,"score":27.9,"version":"Claude 4.1"},{"company":"opengvlab","family":"Intern","model":"internvl3-78b","place":17,"score":27.7,"version":"InternVL 3"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-424b-a47b","place":18,"score":27.3,"version":"ERNIE 4.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":19,"score":25.6,"version":"Qwen2.5 VL"}],"stars":"","top_company":"baidu","top_model":"ERNIE-4.5-VL-28B-A3B-Thinking","top_model_family":"ERNIE","top_model_id":"ernie-4.5-vl-28b-a3b-thinking","top_model_version":"ERNIE 4.5 VL","top_open_source":1,"top_percent":52.5,"topic":"Visual logical reasoning","total_models":1613,"url":"https://visulogic-benchmark.github.io/VisuLogic/"},{"bench_models":27,"description":"Industry-focused benchmark evaluating domain QA performance.","github":"","has_recent":0,"id":"vitabench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VitaBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":1,"score":56.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":51.6,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":3,"score":49.7,"version":"Qwen3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":4,"score":44.1,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":5,"score":41.9,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":6,"score":41.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":7,"score":40.9,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking","place":8,"score":38.2,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":9,"score":38.2,"version":"GPT-5.2"},{"company":"openai","family":"o3","model":"o3","place":10,"score":35.3,"version":"o3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":11,"score":33.6,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":12,"score":31.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":13,"score":31.6,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":14,"score":31.5,"version":"Gemini 3"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking","place":15,"score":29.5,"version":"LongCat-Flash"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":16,"score":29.3,"version":"GPT-5"},{"company":"meituan","family":"LongCat","model":"longcat-flash-thinking-2601","place":17,"score":29.3,"version":"LongCat-Flash"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":18,"score":28.5,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":19,"score":26.8,"version":"GLM 4.5"},{"company":"meituan","family":"LongCat","model":"longcat-flash","place":20,"score":24.3,"version":"LongCat-Flash"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":21,"score":24.3,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":22,"score":24.3,"version":"GPT-5.2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":23,"score":24,"version":"DeepSeek V3.2"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":24,"score":21.5,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.7-thinking","place":25,"score":18.3,"version":"GLM 4.7"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":26,"score":14.5,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":27,"score":13.9,"version":"GPT-5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":28,"score":13.5,"version":"DeepSeek V3.1"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":29,"score":12.8,"version":"Kimi K2"}],"stars":"","top_company":"anthropic","top_model":"Claude Opus 4.5","top_model_family":"Claude","top_model_id":"claude-opus-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":56.3,"topic":"Industry QA","total_models":1613,"url":""},{"bench_models":7,"description":"Reward alignment benchmark for VLMs.","github":"","has_recent":0,"id":"vl-rewardbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VL-RewardBench","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":1,"score":67.4,"version":"Claude 3.7"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":2,"score":62.7,"version":"MiMo VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":3,"score":62.4,"version":"GPT-4o"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":4,"score":61.9,"version":"MiMo VL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":5,"score":51.9,"version":"Gemma 3"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":6,"score":49.7,"version":"InternVL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":7,"score":47.3,"version":"Qwen2.5 VL"}],"stars":"","top_company":"anthropic","top_model":"Claude 3.7 Sonnet","top_model_family":"Claude","top_model_id":"claude-3.7-sonnet","top_model_version":"Claude 3.7","top_open_source":0,"top_percent":67.4,"topic":"Reward modeling (VL)","total_models":1613,"url":""},{"bench_models":5,"description":"Evaluates whether VLMs truly 'see' vs. relying on memorized knowledge; measures bias toward non-visual priors.","github":"https://github.com/anvo25/vlms-are-biased","has_recent":0,"id":"vlms-are-biased","leaderboard":"https://vlmsarebiased.github.io/","lower_is_better":false,"metric":"percent","name":"VLMs are Biased","paper":"https://arxiv.org/abs/2505.23941","placements":[{"company":"openai","family":"o4","model":"o4-mini","place":1,"score":20.25,"version":"o4"},{"company":"openai","family":"o3","model":"o3","place":2,"score":18.5,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":3,"score":16.59,"version":"Claude 3.7"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":16.02,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":5,"score":13.88,"version":"GPT-4.1"}],"stars":"90","top_company":"openai","top_model":"o4 mini","top_model_family":"o4","top_model_id":"o4-mini","top_model_version":"o4","top_open_source":0,"top_percent":20.25,"topic":"Multimodal bias","total_models":1613,"url":"https://vlmsarebiased.github.io/"},{"bench_models":13,"description":"Evaluates failure modes of VLMs in grounding and perception tasks.","github":"https://github.com/anguyen8/vision-llms-are-blind","has_recent":0,"id":"vlms-are-blind","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VLMs are Blind","paper":"","placements":[{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":1,"score":79.4,"version":"MiMo VL"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":2,"score":78,"version":"MiMo VL"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5-vl-28b-a3b-thinking","place":3,"score":77.3,"version":"ERNIE 4.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":76.5,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":5,"score":72.1,"version":"Claude 3.7"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":6,"score":69.6,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":7,"score":65,"version":"Gemini 1.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-8b","place":8,"score":61,"version":"Keye-VL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":9,"score":49.8,"version":"GPT-4o"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":10,"score":45.89,"version":"Claude 3.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":11,"score":37.4,"version":"Qwen2.5 VL"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":12,"score":36.8,"version":"InternVL"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":13,"score":18.6,"version":"Gemma 3"}],"stars":"","top_company":"xiaomi","top_model":"MiMo-VL 7B-RL","top_model_family":"MiMo","top_model_id":"mimo-vl-7b-rl","top_model_version":"MiMo VL","top_open_source":1,"top_percent":79.4,"topic":"Visual grounding robustness","total_models":1613,"url":"https://vlmsareblind.github.io/"},{"bench_models":6,"description":"Benchmark evaluating biases in vision-language models.","github":"","has_recent":0,"id":"vlmsarebiased","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VLMsAreBiased","paper":"","placements":[{"company":"bytedance","family":"Seed","model":"seed1.8","place":1,"score":62,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":50.6,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":30.2,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":4,"score":26.5,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":24.3,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":6,"score":21.7,"version":"Claude 4.5"}],"stars":"","top_company":"bytedance","top_model":"Seed1.8","top_model_family":"Seed","top_model_id":"seed1.8","top_model_version":"Seed 1.8","top_open_source":0,"top_percent":62,"topic":"Multimodal bias","total_models":1613,"url":""},{"bench_models":11,"description":"Benchmark probing robustness of vision-language models to visual perturbations.","github":"","has_recent":0,"id":"vlmsareblind","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VLMsAreBlind","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":97.5,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":2,"score":97,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":3,"score":96.9,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":4,"score":96.7,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":5,"score":93,"version":"Seed 1.8"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":92.1,"version":"Seed 1.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":7,"score":85.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":8,"score":84.3,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":9,"score":84.2,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":10,"score":80.9,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":11,"score":79.5,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":12,"score":75.8,"version":"GPT-5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":97.5,"topic":"Multimodal robustness","total_models":1613,"url":""},{"bench_models":13,"description":"VoiceBench adversarial safety evaluation.","github":"","has_recent":0,"id":"voicebench-advbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench AdvBench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":1,"score":99.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":2,"score":99.2,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":3,"score":99.04,"version":"LFM2.5-Audio"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":4,"score":98.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":5,"score":98.9,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":6,"score":98.7,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":98.1,"version":"Gemini 2.5"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":8,"score":97.31,"version":"LFM2-Audio"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":9,"score":97.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":10,"score":96.9,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":11,"score":88.46,"version":"Qwen2.5 Omni"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":12,"score":57.5,"version":"Mini-Omni2"},{"company":"kyutai","family":"Moshi","model":"moshi","place":13,"score":44.23,"version":"Moshi"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Omni-30B-A3B-Thinking","top_model_family":"Qwen","top_model_id":"qwen3-omni-30b-a3b-thinking","top_model_version":"Qwen3","top_open_source":1,"top_percent":99.4,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":8,"description":"VoiceBench evaluation on AlpacaEval instructions.","github":"","has_recent":0,"id":"voicebench-alpacaeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench AlpacaEval","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":1,"score":96.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":2,"score":96.4,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":3,"score":96.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":4,"score":95.6,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":5,"score":95.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":6,"score":94.8,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":94.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":8,"score":89.9,"version":"Qwen2.5 Omni"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Omni-Flash-Thinking","top_model_family":"Qwen","top_model_id":"qwen3-omni-flash-thinking","top_model_version":"Qwen3","top_open_source":1,"top_percent":96.8,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":8,"description":"VoiceBench evaluation on Big-Bench Hard prompts.","github":"","has_recent":0,"id":"voicebench-bbh","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench BBH","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":92.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":2,"score":89.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":3,"score":88.9,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":4,"score":84.1,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":5,"score":83,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":6,"score":80.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":7,"score":80.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":8,"score":66.7,"version":"Qwen2.5 Omni"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":92.6,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":13,"description":"VoiceBench evaluation on CommonEval.","github":"","has_recent":0,"id":"voicebench-commoneval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench CommonEval","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":1,"score":91,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":2,"score":90.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":3,"score":90.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":4,"score":90.5,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":5,"score":89.8,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":88.4,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":7,"score":88.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":8,"score":76.7,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":9,"score":3.53,"version":"LFM2.5-Audio"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":10,"score":3.51,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":11,"score":3.49,"version":"LFM2-Audio"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":12,"score":2.18,"version":"Mini-Omni2"},{"company":"kyutai","family":"Moshi","model":"moshi","place":13,"score":1.6,"version":"Moshi"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Omni-Flash-Instruct","top_model_family":"Qwen","top_model_id":"qwen3-omni-flash-instruct","top_model_version":"Qwen3","top_open_source":1,"top_percent":91,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":8,"description":"VoiceBench instruction-following evaluation (IFEval).","github":"","has_recent":0,"id":"voicebench-ifeval","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench IFEval","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":85.7,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":2,"score":83.8,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":3,"score":80.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":4,"score":80.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":5,"score":77.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":6,"score":76,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":7,"score":75.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":8,"score":53.5,"version":"Qwen2.5 Omni"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":85.7,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":8,"description":"Audio reasoning benchmark MMAU v05.15.25.","github":"","has_recent":0,"id":"voicebench-mau-v05-15-25","leaderboard":"","lower_is_better":false,"metric":"percent","name":"MMAU v05.15.25","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":1,"score":77.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":2,"score":77.5,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":77.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":4,"score":76.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":5,"score":75.4,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":6,"score":71.8,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":7,"score":65.5,"version":"Qwen2.5 Omni"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":8,"score":62.5,"version":"GPT-4o"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Omni-Flash-Instruct","top_model_family":"Qwen","top_model_id":"qwen3-omni-flash-instruct","top_model_version":"Qwen3","top_open_source":1,"top_percent":77.6,"topic":"Audio reasoning","total_models":1613,"url":""},{"bench_models":13,"description":"VoiceBench MMSU benchmark (voice modality).","github":"","has_recent":0,"id":"voicebench-mmsu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench MMSU","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":1,"score":84.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":2,"score":83,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":3,"score":80.3,"version":"GPT-4o"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":71.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":5,"score":68.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":6,"score":68.1,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":7,"score":66.1,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":8,"score":61.7,"version":"Qwen2.5 Omni"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":9,"score":55.29,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":10,"score":33.04,"version":"LFM2.5-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":11,"score":31.95,"version":"LFM2-Audio"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":12,"score":24.27,"version":"Mini-Omni2"},{"company":"kyutai","family":"Moshi","model":"moshi","place":13,"score":24.04,"version":"Moshi"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Omni-Flash-Thinking","top_model_family":"Qwen","top_model_id":"qwen3-omni-flash-thinking","top_model_version":"Qwen3","top_open_source":1,"top_percent":84.3,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":8,"description":"Audio reasoning MMSU results.","github":"","has_recent":0,"id":"voicebench-mmsu-audio","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench MMSU (Audio)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":77.7,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":2,"score":71.3,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":3,"score":70.2,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":4,"score":70.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":5,"score":69.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":6,"score":69,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":7,"score":62.6,"version":"Qwen2.5 Omni"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":8,"score":56.4,"version":"GPT-4o"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":77.7,"topic":"Audio reasoning","total_models":1613,"url":""},{"bench_models":8,"description":"VoiceBench results on OpenBookQA prompts.","github":"","has_recent":0,"id":"voicebench-openbookqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench OpenBookQA","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":1,"score":95,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":2,"score":94.3,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":92.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":4,"score":91.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":5,"score":89.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":6,"score":89.2,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":7,"score":89,"version":"Qwen2.5 Omni"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":8,"score":56.9,"version":"Gemini 2.5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-Omni-Flash-Thinking","top_model_family":"Qwen","top_model_id":"qwen3-omni-flash-thinking","top_model_version":"Qwen3","top_open_source":1,"top_percent":95,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":13,"description":"VoiceBench Spoken Dialogue QA results.","github":"","has_recent":0,"id":"voicebench-sd-qa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench SD-QA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":90.1,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":2,"score":84.5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":3,"score":78.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":4,"score":78.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":5,"score":76.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":6,"score":76.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":7,"score":75.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":8,"score":56.4,"version":"Qwen2.5 Omni"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":9,"score":44.94,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":10,"score":30.92,"version":"LFM2.5-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":11,"score":30.56,"version":"LFM2-Audio"},{"company":"kyutai","family":"Moshi","model":"moshi","place":12,"score":15.64,"version":"Moshi"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":13,"score":9.31,"version":"Mini-Omni2"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":90.1,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":13,"description":"VoiceBench evaluation on WildVoice dataset.","github":"","has_recent":0,"id":"voicebench-wildvoice","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VoiceBench WildVoice","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":1,"score":93.4,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":2,"score":92.3,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":3,"score":92.1,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-audio","place":4,"score":91.6,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":5,"score":91.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":6,"score":90.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":7,"score":90.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni","place":8,"score":77.7,"version":"Qwen2.5 Omni"},{"company":"qwen","family":"Qwen","model":"qwen2.5-omni-3b","place":9,"score":3.42,"version":"Qwen2.5 Omni"},{"company":"liquidai","family":"LFM","model":"lfm2-audio-1.5b","place":10,"score":3.17,"version":"LFM2-Audio"},{"company":"liquidai","family":"LFM","model":"lfm2.5-audio-1.5b","place":11,"score":3.15,"version":"LFM2.5-Audio"},{"company":"gpt4o-omni","family":"Mini-Omni","model":"mini-omni2","place":12,"score":1.79,"version":"Mini-Omni2"},{"company":"kyutai","family":"Moshi","model":"moshi","place":13,"score":1.3,"version":"Moshi"}],"stars":"","top_company":"google","top_model":"Gemini 2.5 Pro","top_model_family":"Gemini","top_model_id":"gemini-2.5-pro","top_model_version":"Gemini 2.5","top_open_source":0,"top_percent":93.4,"topic":"VoiceBench","total_models":1613,"url":""},{"bench_models":6,"description":"Visual perception and comprehension test.","github":"","has_recent":0,"id":"vpct","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VPCT","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":90,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":61,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":3,"score":56,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":4,"score":52,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":5,"score":41,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":6,"score":35,"version":"Seed 1.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":90,"topic":"Multimodal reasoning","total_models":1613,"url":""},{"bench_models":52,"description":"Standard Visual Question Answering v2 benchmark on natural images.","github":"","has_recent":0,"id":"vqav2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VQAv2","paper":"","placements":[{"company":"allenai","family":"Molmo2","model":"molmo2-8b","place":1,"score":87,"version":"Molmo2"},{"company":"allenai","family":"Molmo2","model":"molmo2-4b","place":2,"score":86.6,"version":"Molmo2"},{"company":"allenai","family":"Molmo2","model":"molmo2-o-7b","place":3,"score":86.6,"version":"Molmo2"},{"company":"allenai","family":"Molmo","model":"molmo-72b","place":4,"score":86.5,"version":"Molmo"},{"company":"opengvlab","family":"Intern","model":"internvl2-llama-3-76b","place":5,"score":85.6,"version":"InternVL 2"},{"company":"allenai","family":"Molmo","model":"molmo-7b-d","place":6,"score":85.6,"version":"Molmo"},{"company":"unknown","family":"PLM","model":"plm-8b","place":7,"score":85.6,"version":"PLM"},{"company":"allenai","family":"Molmo","model":"molmo-7b-o","place":8,"score":85.3,"version":"Molmo"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":9,"score":85.2,"version":"LLaVA OneVision"},{"company":"unknown","family":"PLM","model":"plm-3b","place":10,"score":84.4,"version":"PLM"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-7b","place":11,"score":84,"version":"LLaVA OneVision"},{"company":"allenai","family":"Molmo","model":"molmoe-1b","place":12,"score":83.9,"version":"MolmoE"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-34b","place":13,"score":83.8,"version":"Cambrian 1"},{"company":"microsoft","family":"LLaVA","model":"llava-onevision-72b","place":14,"score":83.8,"version":"LLaVA OneVision"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-7b","place":15,"score":82.9,"version":"Qwen2 VL"},{"company":"unknown","family":"Eagle","model":"eagle2.5-8b","place":16,"score":82.4,"version":"Eagle 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":17,"score":82.3,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2-vl-72b","place":18,"score":81.9,"version":"Qwen2 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":19,"score":81.7,"version":"Qwen3 VL"},{"company":"salesforce","family":"xGen","model":"xgen-mm-interleave-4b","place":20,"score":81.5,"version":"xGen-MM"},{"company":"nyu","family":"Cambrian","model":"cambrian-1-8b","place":21,"score":81.2,"version":"Cambrian 1"},{"company":"mistral","family":"Pixtral","model":"pixtral-large-2411","place":22,"score":80.9,"version":"Pixtral"},{"company":"apple","family":"FastVLM","model":"fastvlm-7b","place":23,"score":80.8,"version":"FastVLM"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro","place":24,"score":80.2,"version":"Gemini 1.5"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash","place":25,"score":80.1,"version":"Gemini 1.5"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-13b","place":26,"score":80,"version":"LLaVA 1.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":27,"score":79.7,"version":"GPT-5"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-8b","place":28,"score":79.5,"version":"InternVL 3.5"},{"company":"kuaishou","family":"Keye-VL","model":"keye-vl-1.5-8b","place":29,"score":79.3,"version":"Keye-VL 1.5"},{"company":"apple","family":"FastVLM","model":"fastvlm-1.5b","place":30,"score":79.1,"version":"FastVLM"},{"company":"openai","family":"GPT","model":"gpt-4o-0513","place":31,"score":78.7,"version":"GPT-4o"},{"company":"mistral","family":"Pixtral","model":"pixtral-12b","place":32,"score":78.6,"version":"Pixtral"},{"company":"microsoft","family":"LLaVA","model":"llava-1.5-7b","place":33,"score":78.5,"version":"LLaVA 1.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-90b-instruct","place":34,"score":78.1,"version":"Llama 3.2"},{"company":"opengvlab","family":"Intern","model":"internvl3.5-4b","place":35,"score":78.1,"version":"InternVL 3.5"},{"company":"openai","family":"GPT","model":"gpt-4o","place":36,"score":77.8,"version":"GPT-4o"},{"company":"openai","family":"GPT","model":"gpt-4v","place":37,"score":77.2,"version":"GPT-4V"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":38,"score":77,"version":"Claude 4.5"},{"company":"opengvlab","family":"Intern","model":"internvl2-8b","place":39,"score":76.7,"version":"InternVL 2"},{"company":"apple","family":"FastVLM","model":"fastvlm-0.5b","place":40,"score":76.3,"version":"FastVLM"},{"company":"google","family":"PaliGemma","model":"paligemma-mix-3b","place":41,"score":76.3,"version":"PaliGemma"},{"company":"microsoft","family":"Phi","model":"phi-3.5-vision-4b","place":42,"score":75.7,"version":"Phi 3.5"},{"company":"meta","family":"Llama","model":"llama-3.2v-11b-instruct","place":43,"score":75.2,"version":"Llama 3.2"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":44,"score":74.1,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":45,"score":72.1,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-3.5-sonnet","place":46,"score":70.7,"version":"Claude 3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":47,"score":69.4,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-haiku","place":48,"score":68.4,"version":"Claude 3"},{"company":"zai","family":"GLM","model":"glm-4.1v-9b","place":49,"score":68.3,"version":"GLM 4.1V"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":50,"score":67.1,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-3-opus","place":51,"score":66.3,"version":"Claude 3"},{"company":"google","family":"Gemini","model":"gemini-1.5-flash-8b-0827","place":52,"score":65.5,"version":"Gemini 1.5"},{"company":"openbmb","family":"MiniCPM","model":"minicpm-v-4.5-8b","place":53,"score":64.1,"version":"MiniCPM V 4.5"}],"stars":"","top_company":"allenai","top_model":"Molmo2-8B","top_model_family":"Molmo2","top_model_id":"molmo2-8b","top_model_version":"Molmo2","top_open_source":1,"top_percent":87,"topic":"Visual question answering","total_models":1613,"url":"https://visualqa.org/"},{"bench_models":11,"description":"Visual spatial intelligence benchmark covering 3D reasoning and spatial inference tasks.","github":"","has_recent":0,"id":"vsi-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"VSI-Bench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":1,"score":63.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":2,"score":61.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":3,"score":59.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":4,"score":58.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":5,"score":53.9,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":6,"score":45.8,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":7,"score":37.7,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":8,"score":36.1,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":9,"score":30.5,"version":"Qwen2.5 VL"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":10,"score":27,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":11,"score":27,"version":"GPT-5"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL-30B-A3B Instruct","top_model_family":"Qwen","top_model_id":"qwen3-vl-30b-a3b-instruct","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":63.2,"topic":"Spatial intelligence","total_models":1613,"url":""},{"bench_models":12,"description":"Task success on the WebClick UI agent benchmark.","github":"","has_recent":0,"id":"webclick","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WebClick","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":1,"score":93,"version":"Claude 4"},{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":2,"score":92.43,"version":"Holo1.5"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":3,"score":90.24,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":4,"score":88.29,"version":"Qwen2.5 VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":5,"score":86.1,"version":"UI-TARS"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":6,"score":84.44,"version":"UI-Venus"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":7,"score":84.04,"version":"Holo1"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":8,"score":81.45,"version":"Holo1.5"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":9,"score":79.35,"version":"Holo1"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":10,"score":77,"version":"UI-Venus"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":11,"score":76.51,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":12,"score":71.2,"version":"Qwen2.5 VL"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4","top_model_family":"Claude","top_model_id":"claude-sonnet-4","top_model_version":"Claude 4","top_open_source":0,"top_percent":93,"topic":"GUI agents","total_models":1613,"url":""},{"bench_models":52,"description":"Arena evaluation for autonomous web development agents.","github":"","has_recent":0,"id":"webdev-arena","leaderboard":"https://lmarena.ai/leaderboard/webdev","lower_is_better":false,"metric":"elo","name":"WebDev Arena","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5","place":1,"score":1483,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":2,"score":1480,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":3,"score":1476,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805-thinking-16k","place":4,"score":1468,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-1-20250805","place":5,"score":1458,"version":"Claude 4.1"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":1405,"version":"Gemini 2.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":7,"score":1403,"version":"Gemini 2.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":8,"score":1393,"version":"DeepSeek R1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":9,"score":1393,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":10,"score":1383,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-opus-4-20250514","place":11,"score":1383,"version":"Claude 4"},{"company":"zai","family":"GLM","model":"glm-4.5","place":12,"score":1375,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5","place":13,"score":1371,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b","place":14,"score":1367,"version":"Qwen3"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":15,"score":1367,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":16,"score":1366,"version":"GLM 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder","place":17,"score":1366,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":18,"score":1364,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1-thinking","place":19,"score":1362,"version":"DeepSeek V3.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4-20250514","place":20,"score":1362,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":21,"score":1360,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-3-7-sonnet-20250219","place":22,"score":1358,"version":"Claude 3.7"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":23,"score":1340,"version":"DeepSeek V3.1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-plus-2025-09-23","place":24,"score":1332,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":25,"score":1316,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":26,"score":1289,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1-2025-04-14","place":27,"score":1253,"version":"GPT-4.1"},{"company":"anthropic","family":"Claude","model":"claude-3-5-sonnet-20241022","place":28,"score":1238,"version":"Claude 3.5"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":29,"score":1208,"version":"DeepSeek V3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":30,"score":1199,"version":"DeepSeek R1"},{"company":"openai","family":"GPT","model":"gpt-4.1-mini-2025-04-14","place":31,"score":1193,"version":"GPT-4.1"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":32,"score":1189,"version":"Qwen3"},{"company":"openai","family":"o3","model":"o3-2025-04-16","place":33,"score":1186,"version":"o3"},{"company":"mistral","family":"Mistral","model":"mistral-medium-3","place":34,"score":1181,"version":"Mistral 3"},{"company":"xai","family":"Grok","model":"grok-4-0709","place":35,"score":1174,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-code-fast-1","place":36,"score":1150,"version":"Grok Code"},{"company":"xai","family":"Grok","model":"grok-3-preview-02-24","place":37,"score":1143,"version":"Grok 3"},{"company":"openai","family":"o3","model":"o3-mini-high","place":38,"score":1137,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-3-5-haiku-20241022","place":39,"score":1133,"version":"Claude 3.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m1","place":40,"score":1129,"version":"MiniMax M1"},{"company":"openai","family":"o4","model":"o4-mini-2025-04-16","place":41,"score":1118,"version":"o4"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":42,"score":1094,"version":"GPT-OSS"},{"company":"openai","family":"o3","model":"o3-mini","place":43,"score":1092,"version":"o3"},{"company":"google","family":"Gemini","model":"gemini-2.0-pro-exp-02-05","place":44,"score":1090,"version":"Gemini 2.0"},{"company":"openai","family":"o1","model":"o1-2024-12-17","place":45,"score":1045,"version":"o1"},{"company":"openai","family":"o1","model":"o1-mini","place":46,"score":1043,"version":"o1"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-001","place":47,"score":1040,"version":"Gemini 2.0"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-thinking-01-21","place":48,"score":1030,"version":"Gemini 2.0"},{"company":"meta","family":"Llama","model":"llama-4-maverick-17b-128e-instruct","place":49,"score":1027,"version":"Llama 4"},{"company":"google","family":"Gemini","model":"gemini-2.0-flash-exp","place":50,"score":980,"version":"Gemini 2.0"},{"company":"qwen","family":"Qwen","model":"qwen2.5-max","place":51,"score":976,"version":"Qwen2.5"},{"company":"openai","family":"GPT","model":"gpt-4o-2024-11-20","place":52,"score":964,"version":"GPT-4o"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":53,"score":960,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":54,"score":902,"version":"Qwen2.5"},{"company":"meta","family":"Llama","model":"llama-4-scout-17b-16e-instruct","place":55,"score":901,"version":"Llama 4"},{"company":"google","family":"Gemini","model":"gemini-1.5-pro-002","place":56,"score":893,"version":"Gemini 1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-instruct","place":57,"score":810,"version":"Llama 3.1"}],"stars":"","top_company":"openai","top_model":"GPT-5","top_model_family":"GPT","top_model_id":"gpt-5","top_model_version":"GPT-5","top_open_source":0,"top_percent":1483,"topic":"Web development agents","total_models":1613,"url":""},{"bench_models":7,"description":"Multi-question web search and interaction tasks.","github":"","has_recent":0,"id":"webquest-multiqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WebQuest-MultiQA","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.5v","place":1,"score":60.6,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":2,"score":59,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":3,"score":54.7,"version":"GLM 4.1V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":4,"score":53.4,"version":"GLM 4.6V"},{"company":"stepfun","family":"Step","model":"step-3","place":5,"score":52.8,"version":"Step 3"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":6,"score":36.5,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":7,"score":11.1,"version":"Kimi VL"}],"stars":"","top_company":"zai","top_model":"GLM-4.5V","top_model_family":"GLM","top_model_id":"glm-4.5v","top_model_version":"GLM 4.5V","top_open_source":1,"top_percent":60.6,"topic":"Web agents","total_models":1613,"url":""},{"bench_models":7,"description":"Single-question web search and interaction tasks.","github":"","has_recent":0,"id":"webquest-singleqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WebQuest-SingleQA","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.6v","place":1,"score":79.5,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":2,"score":76.9,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":3,"score":75.1,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":4,"score":72.1,"version":"GLM 4.1V"},{"company":"stepfun","family":"Step","model":"step-3","place":5,"score":58.7,"version":"Step 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":6,"score":35.6,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":7,"score":31.2,"version":"Gemma 3"}],"stars":"","top_company":"zai","top_model":"GLM-4.6V","top_model_family":"GLM","top_model_id":"glm-4.6v","top_model_version":"GLM 4.6V","top_open_source":1,"top_percent":79.5,"topic":"Web agents","total_models":1613,"url":""},{"bench_models":18,"description":"Webpage question answering (SQuAD F1).","github":"","has_recent":0,"id":"websrc","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WebSrc","paper":"","placements":[{"company":"hcompany","family":"Holo","model":"holo1.5-72b","place":1,"score":97.2,"version":"Holo1.5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":2,"score":97,"version":"Qwen2.5 VL"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-72b","place":3,"score":96.7,"version":"UI-Venus"},{"company":"antgroup","family":"UI-Venus","model":"ui-venus-7b","place":4,"score":96.6,"version":"UI-Venus"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-sft","place":5,"score":96.5,"version":"MiMo VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":6,"score":96,"version":"Claude 4"},{"company":"hcompany","family":"Holo","model":"holo1.5-7b","place":7,"score":95.9,"version":"Holo1.5"},{"company":"hcompany","family":"Holo","model":"holo1-7b","place":8,"score":95.3,"version":"Holo1"},{"company":"xiaomi","family":"MiMo","model":"mimo-vl-7b-rl","place":9,"score":95.3,"version":"MiMo VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-7b","place":10,"score":95,"version":"Qwen2.5 VL"},{"company":"hcompany","family":"Holo","model":"holo1.5-3b","place":11,"score":94.8,"version":"Holo1.5"},{"company":"hcompany","family":"Holo","model":"holo1-3b","place":12,"score":93.9,"version":"Holo1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-3b","place":13,"score":93,"version":"Qwen2.5 VL"},{"company":"bytedance","family":"UI-TARS","model":"ui-tars-1.5-7b","place":14,"score":92.9,"version":"UI-TARS"},{"company":"anthropic","family":"Claude","model":"claude-3.7-sonnet","place":15,"score":91.1,"version":"Claude 3.7"},{"company":"opengvlab","family":"Intern","model":"internvl-8b","place":16,"score":91.1,"version":"InternVL"},{"company":"openai","family":"GPT","model":"gpt-4o","place":17,"score":89.1,"version":"GPT-4o"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":18,"score":89,"version":"Gemma 3"}],"stars":"","top_company":"hcompany","top_model":"Holo1.5-72B","top_model_family":"Holo","top_model_id":"holo1.5-72b","top_model_version":"Holo1.5","top_open_source":1,"top_percent":97.2,"topic":"Web QA","total_models":1613,"url":""},{"bench_models":2,"description":"Web navigation and interaction tasks for LLM agents.","github":"","has_recent":0,"id":"webvoyager","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WebVoyager","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.6v","place":1,"score":81,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":2,"score":71.8,"version":"GLM 4.6V"}],"stars":"","top_company":"zai","top_model":"GLM-4.6V","top_model_family":"GLM","top_model_id":"glm-4.6v","top_model_version":"GLM 4.6V","top_open_source":1,"top_percent":81,"topic":"Web agents","total_models":1613,"url":""},{"bench_models":3,"description":"Web navigation and interaction tasks for LLM agents (v2).","github":"","has_recent":0,"id":"webvoyager2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WebVoyager2","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.5v","place":1,"score":84.4,"version":"GLM 4.5V"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":2,"score":69,"version":"GLM 4.1V"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":3,"score":34.8,"version":"Gemma 3"}],"stars":"","top_company":"zai","top_model":"GLM-4.5V","top_model_family":"GLM","top_model_id":"glm-4.5v","top_model_version":"GLM 4.5V","top_open_source":1,"top_percent":84.4,"topic":"Web agents","total_models":1613,"url":""},{"bench_models":6,"description":"WebWalker tasks evaluating autonomous browsing question answering performance.","github":"","has_recent":0,"id":"webwalker-qa","leaderboard":"","lower_is_better":false,"metric":"avg@3","name":"WebWalkerQA","paper":"","placements":[{"company":"tongyi","family":"Tongyi","model":"tongyi-deepresearch","place":1,"score":72.2,"version":"Tongyi"},{"company":"openai","family":"o3","model":"o3","place":2,"score":71.7,"version":"o3"},{"company":"zai","family":"GLM","model":"glm-4.5","place":3,"score":65.6,"version":"GLM 4.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":4,"score":63,"version":"Kimi K2"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":5,"score":61.7,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":6,"score":61.2,"version":"DeepSeek V3.1"}],"stars":"","top_company":"tongyi","top_model":"Tongyi DeepResearch","top_model_family":"Tongyi","top_model_id":"tongyi-deepresearch","top_model_version":"Tongyi","top_open_source":0,"top_percent":72.2,"topic":"Web agents","total_models":1613,"url":""},{"bench_models":17,"description":"Math reasoning benchmark spanning diverse curricula and difficulty levels.","github":"","has_recent":0,"id":"wemath","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WeMath","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":87.9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":86.9,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":84.7,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":79,"version":"GPT-5.2"},{"company":"alibaba","family":"Ovis","model":"ovis2.6-30b-a3b","place":5,"score":75,"version":"Ovis 2.6"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":74.9,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":7,"score":74.8,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":8,"score":70,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":9,"score":69.8,"version":"GLM 4.6V"},{"company":"zai","family":"GLM","model":"glm-4.5v","place":10,"score":68.8,"version":"GLM 4.5V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":11,"score":66.1,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":12,"score":65.1,"version":"GPT-5"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":13,"score":63.8,"version":"GLM 4.1V"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":14,"score":60,"version":"GLM 4.6V"},{"company":"stepfun","family":"Step","model":"step-3","place":15,"score":59.8,"version":"Step 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":16,"score":42,"version":"Kimi VL"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":17,"score":37.9,"version":"Gemma 3"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":87.9,"topic":"Math reasoning","total_models":1613,"url":""},{"bench_models":18,"description":"Wide web search and QA benchmark.","github":"","has_recent":0,"id":"wide-search","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WideSearch","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5.2","place":1,"score":76.8,"version":"GPT-5.2"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":2,"score":76.4,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":3,"score":76.2,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":4,"score":74,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":5,"score":72.7,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":6,"score":72.7,"version":"Kimi K2.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":7,"score":68,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":8,"score":65.7,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":9,"score":63.8,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":10,"score":62.2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":11,"score":61.1,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":12,"score":60.5,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":13,"score":57.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":14,"score":57.1,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":57,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":16,"score":52.6,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":17,"score":47.2,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":18,"score":40.4,"version":"GPT-OSS"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":19,"score":32.5,"version":"DeepSeek V3.2"}],"stars":"","top_company":"openai","top_model":"GPT-5.2","top_model_family":"GPT","top_model_id":"gpt-5.2","top_model_version":"GPT-5.2","top_open_source":0,"top_percent":76.8,"topic":"Web search","total_models":1613,"url":""},{"bench_models":5,"description":"Adversarial jailbreak benchmark evaluating refusal robustness.","github":"","has_recent":0,"id":"wild-jailbreak","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Wild-Jailbreak","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":1,"score":98.2,"version":"GPT-OSS"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":2,"score":89.9,"version":"K-EXAONE"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":3,"score":85.5,"version":"Qwen3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":79.1,"version":"DeepSeek V3.2"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":5,"score":62.8,"version":"EXAONE 4.0"}],"stars":"","top_company":"openai","top_model":"GPT-OSS 120B (High)","top_model_family":"GPT","top_model_id":"gpt-oss-120b-high","top_model_version":"GPT-OSS","top_open_source":1,"top_percent":98.2,"topic":"Safety / jailbreak","total_models":1613,"url":""},{"bench_models":2,"description":"WildBench V2 human preference benchmark for instruction following and helpfulness.","github":"","has_recent":0,"id":"wildbench-v2","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WildBench V2","paper":"","placements":[{"company":"mistral","family":"Mistral","model":"mistral-small-3.2-24b-instruct","place":1,"score":65.33,"version":"Mistral 3.2"},{"company":"mistral","family":"Mistral","model":"mistral-small-3.1-24b-instruct","place":2,"score":55.6,"version":"Mistral 3.1"}],"stars":"","top_company":"mistral","top_model":"Mistral Small 3.2 24B Instruct","top_model_family":"Mistral","top_model_id":"mistral-small-3.2-24b-instruct","top_model_version":"Mistral 3.2","top_open_source":1,"top_percent":65.33,"topic":"Instruction following","total_models":1613,"url":""},{"bench_models":4,"description":"WildGuardTest safety benchmark.","github":"","has_recent":0,"id":"wildguardtest","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WildGuardTest","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":1,"score":86.8,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":2,"score":85,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":3,"score":78.1,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":4,"score":73.9,"version":"Qwen2.5"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Thinking","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-thinking","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":86.8,"topic":"Safety","total_models":1613,"url":""},{"bench_models":3,"description":"Coreference resolution dataset for measuring gender bias.","github":"https://github.com/rudinger/winogender-schemas","has_recent":0,"id":"winogender","leaderboard":"","lower_is_better":false,"metric":"percent","name":"Winogender","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":1,"score":84.3,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":2,"score":67.9,"version":"Llama TFree HAT"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":3,"score":67.9,"version":"Llama TFree HAT"},{"company":"alephalpha","family":"TFree","model":"tfree-hat-pretrained-7b-base","place":4,"score":67.1,"version":"TFree Hat"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B Instruct","top_model_family":"Llama","top_model_id":"llama-3.3-70b-instruct","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":84.3,"topic":"Gender bias (coreference)","total_models":1613,"url":""},{"bench_models":111,"description":"Large-scale adversarial Winograd Schema-style pronoun resolution.","github":"","has_recent":0,"id":"winogrande","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WinoGrande","paper":"","placements":[{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":1,"score":90.3,"version":"OLMo 3"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":2,"score":86.7,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-3.1-405b-base","place":3,"score":86.7,"version":"Llama 3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":4,"score":85.9,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":5,"score":85.6,"version":"DeepSeek V3.2"},{"company":"cohere","family":"Command","model":"command-r-plus-08-2024","place":6,"score":85.4,"version":"Command"},{"company":"meta","family":"Llama 3.0","model":"llama3.0-70b","place":7,"score":85.3,"version":"Llama 3.0"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-base","place":8,"score":85.3,"version":"Kimi K2"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x22b","place":9,"score":84.7,"version":"Mixtral"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":10,"score":83.9,"version":"K2 V2"},{"company":"qwen","family":"Qwen","model":"qwen-2.5-72b","place":11,"score":83.9,"version":"Qwen2.5"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash-base","place":12,"score":83.8,"version":"MiMo V2"},{"company":"upstage","family":"Solar","model":"solar-10.7b-instruct-v1.0","place":13,"score":83.58,"version":"Solar Mini"},{"company":"meta","family":"Llama","model":"llama-3.1-70b-base","place":14,"score":83.3,"version":"Llama 3.1"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":15,"score":83.2,"version":"Llama 2"},{"company":"mbzuai","family":"K2","model":"k2-v2","place":16,"score":82.6,"version":"K2 V2"},{"company":"cohere","family":"Command","model":"command-r-08-2024","place":17,"score":81.5,"version":"Command"},{"company":"moonshotai","family":"Kimi","model":"kimi-linear-base","place":18,"score":81.5,"version":"Kimi Linear"},{"company":"arcee","family":"Trinity","model":"trinity-large-truebase","place":19,"score":81.45,"version":"Trinity Large"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":20,"score":81.2,"version":"Mixtral"},{"company":"google","family":"PaLM","model":"palm-540b","place":21,"score":81.1,"version":"PaLM"},{"company":"arcee","family":"Trinity","model":"trinity-large-base","place":22,"score":80.82,"version":"Trinity Large"},{"company":"meta","family":"Llama","model":"llama-2-70b","place":23,"score":80.4,"version":"Llama 2"},{"company":"meta","family":"Llama 3.1","model":"llama3.1-70b","place":24,"score":79.8,"version":"Llama 3.1"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.5-instruct","place":25,"score":79.72,"version":"Bielik v2.5"},{"company":"nvidia","family":"Nemotron","model":"nemotron-3-nano-30b-a3b-base","place":26,"score":79.64,"version":"Nemotron 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-base","place":27,"score":79.6,"version":"Motif 2"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":28,"score":79.3,"version":"Marin 32B"},{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":29,"score":79,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":30,"score":78.9,"version":"OLMo 2"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":31,"score":78.8,"version":"Gemma 3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.6-instruct","place":32,"score":78.77,"version":"Bielik v2.6"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":33,"score":78.6,"version":"Marin 32B"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0","place":34,"score":78.53,"version":"Bielik v3.0"},{"company":"meta","family":"Llama","model":"llama-3-8b","place":35,"score":78.45,"version":"Llama 3"},{"company":"google","family":"gemma-7b","model":"gemma-7b","place":36,"score":78.45,"version":"gemma-7b"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.1","place":37,"score":78.37,"version":"Mistral v0.1"},{"company":"openchat","family":"OpenChat","model":"openchat-3.5-0106-gemma","place":38,"score":78.3,"version":"OpenChat 3.5 0106"},{"company":"mistral","family":"Mistral","model":"mistral-7b-v0.2","place":39,"score":78.22,"version":"Mistral v0.2"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":40,"score":78,"version":"Mistral"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2","place":41,"score":77.82,"version":"Bielik v2"},{"company":"google","family":"Gemma","model":"gemma-3-it-12b","place":42,"score":77.74,"version":"Gemma 3"},{"company":"mistral","family":"Mixtral","model":"mixtral-8x7b","place":43,"score":77.2,"version":"Mixtral"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v3.0-instruct","place":44,"score":77.19,"version":"Bielik v3.0"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.2","place":45,"score":77.19,"version":"Mistral v0.2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-32b-chat","place":46,"score":77.19,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"llama-3.1-8b","place":47,"score":77.11,"version":"Llama 3.1"},{"company":"google","family":"PaLM","model":"palm-62b","place":48,"score":77,"version":"PaLM"},{"company":"google","family":"PaLM","model":"palm-cont-62b","place":49,"score":77,"version":"PaLM"},{"company":"meta","family":"Llama","model":"llama-1-65b","place":50,"score":77,"version":"Llama 1"},{"company":"qwen","family":"Qwen","model":"qwen3-8b","place":51,"score":76.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-14b","place":52,"score":76.8,"version":"Qwen"},{"company":"qwen","family":"Qwen","model":"qwen2.5-7b","place":53,"score":76.48,"version":"Qwen2.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.0-instruct","place":54,"score":76.32,"version":"Bielik v2.0"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.3-instruct","place":55,"score":76.24,"version":"Bielik v2.3"},{"company":"meta","family":"Llama","model":"llama-1-33b","place":56,"score":76,"version":"Llama 1"},{"company":"bytedance","family":"Ouro","model":"ouro-2.6b","place":57,"score":75.85,"version":"Ouro"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":58,"score":75.7,"version":"Qwen2.5"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":59,"score":75.3,"version":"Mistral"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.2-instruct","place":60,"score":75.3,"version":"Bielik v2.2"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-11b-v2.1-instruct","place":61,"score":75.06,"version":"Bielik v2.1"},{"company":"google","family":"Chinchilla","model":"chinchilla-70b","place":62,"score":74.9,"version":"Chinchilla"},{"company":"moonshotai","family":"Moonlight","model":"moonlight-base","place":63,"score":74.6,"version":"Moonlight"},{"company":"meta","family":"Llama","model":"meta-llama-3-8b-instruct","place":64,"score":74.51,"version":"Llama 3"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":65,"score":74.4,"version":"Nemotron Nano v2"},{"company":"google","family":"Gemma","model":"gemma-3-12b","place":66,"score":74.3,"version":"Gemma 3"},{"company":"mistral","family":"Mistral","model":"mistral-7b","place":67,"score":74.2,"version":"Mistral"},{"company":"mistral","family":"Mistral","model":"mistral-7b-instruct-v0.1","place":68,"score":73.72,"version":"Mistral v0.1"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b","place":69,"score":73.48,"version":"Qwen1.5"},{"company":"qwen","family":"Qwen","model":"qwen1.5-14b-chat","place":70,"score":73.09,"version":"Qwen1.5"},{"company":"meta","family":"Llama","model":"llama-1-13b","place":71,"score":73,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-13b","place":72,"score":72.9,"version":"Llama 2"},{"company":"microsoft","family":"Phi","model":"phi-3-medium-4k-instruct","place":73,"score":72.69,"version":"Phi 3"},{"company":"google","family":"Gemma","model":"gemma-2-7b","place":74,"score":72.3,"version":"Gemma 2"},{"company":"bytedance","family":"Ouro","model":"ouro-1.4b","place":75,"score":72.3,"version":"Ouro"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3","place":76,"score":71.35,"version":"Bielik v3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-tfree-hat-pretrained-7b-dpo","place":77,"score":71.3,"version":"Llama TFree HAT"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":78,"score":71.19,"version":"Qwen3"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-4.5b-v3-instruct","place":79,"score":71.19,"version":"Bielik v3"},{"company":"google","family":"Gemma","model":"gemma-3-it-4b","place":80,"score":71.07,"version":"Gemma 3"},{"company":"openai","family":"GPT","model":"gpt-2","place":81,"score":70.7,"version":"GPT-2"},{"company":"openai","family":"GPT","model":"gpt-3","place":82,"score":70.2,"version":"GPT-3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-3b","place":83,"score":70.17,"version":"Qwen2.5"},{"company":"google","family":"Gopher","model":"gopher-280b","place":84,"score":70.1,"version":"Gopher"},{"company":"meta","family":"Llama","model":"llama-1-7b","place":85,"score":70.1,"version":"Llama 1"},{"company":"meta","family":"Llama","model":"llama-2-7b","place":86,"score":69.5,"version":"Llama 2"},{"company":"meta","family":"Llama","model":"llama-3.2-3b","place":87,"score":69.14,"version":"Llama 3.2"},{"company":"qwen","family":"Qwen","model":"qwen1.5-7b-chat","place":88,"score":67.72,"version":"Qwen1.5"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-v0.1","place":89,"score":66.85,"version":"Bielik v0.1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b","place":90,"score":66.77,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":91,"score":66.3,"version":"Qwen3"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":92,"score":66,"version":"Pythia"},{"company":"SpeakLeash","family":"Bielik","model":"bielik-7b-instruct-v0.1","place":93,"score":65.51,"version":"Bielik v0.1"},{"company":"google","family":"Gemma","model":"gemma-2-2b","place":94,"score":65.4,"version":"Gemma 2"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":95,"score":63.1,"version":"Pythia"},{"company":"meta","family":"MobileLLM","model":"mobilellm-p1-base","place":96,"score":62.83,"version":"MobileLLM P1"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":97,"score":62.75,"version":"Llama 3.2"},{"company":"codellama","family":"CodeLlama-7B","model":"codellama-7b","place":98,"score":62.3,"version":"CodeLlama-7B"},{"company":"meta","family":"Llama","model":"llama-3.2-1b","place":99,"score":61.09,"version":"Llama 3.2"},{"company":"meta","family":"Llama","model":"llama-3.1-8b-base","place":100,"score":60.5,"version":"Llama 3.1"},{"company":"huggingface","family":"SmolLM","model":"smollm2-1.7b-pretrained","place":101,"score":59.4,"version":"SmolLM2"},{"company":"qwen","family":"Qwen","model":"qwen2.5-1.5b-pretrained","place":102,"score":59.3,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-it-1b","place":103,"score":58.72,"version":"Gemma 3"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":104,"score":58.2,"version":"Pythia"},{"company":"google","family":"Gemma","model":"gemma-3-1b","place":105,"score":58.2,"version":"Gemma 3"},{"company":"meta","family":"Llama","model":"llama-1b-pretrained","place":106,"score":57.8,"version":"Llama 1"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":107,"score":56.6,"version":"Pythia"},{"company":"huggingface","family":"SmolLM","model":"smollm1-1.7b-pretrained","place":108,"score":54.7,"version":"SmolLM1"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":109,"score":53.4,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":110,"score":52.9,"version":"Pythia"},{"company":"google","family":"Gemma","model":"gemma-3-it-270m","place":111,"score":52.3,"version":"Gemma 3"},{"company":"google","family":"Gemma","model":"gemma-3-pt-270m","place":112,"score":52,"version":"Gemma 3"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-base","place":113,"score":51.3,"version":"SmolLM2"},{"company":"huggingface","family":"SmolLM","model":"smollm2-135m-pretrained","place":114,"score":51.3,"version":"SmolLM2"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":115,"score":49.7,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":116,"score":49.2,"version":"Pythia"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":117,"score":0.749,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":118,"score":0.697,"version":"Llama 3.3"}],"stars":"99","top_company":"allenai","top_model":"OLMo-3-Think-32B","top_model_family":"OLMo","top_model_id":"olmo-3-think-32b","top_model_version":"OLMo 3","top_open_source":1,"top_percent":90.3,"topic":"Coreference reasoning","total_models":1613,"url":"https://github.com/allenai/winogrande"},{"bench_models":2,"description":"German translation of the WinoGrande pronoun resolution benchmark.","github":"","has_recent":0,"id":"winogrande-de","leaderboard":"","lower_is_better":false,"metric":"norm_log_acc","name":"WinoGrande (DE)","paper":"","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":0.793,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":2,"score":0.761,"version":"Llama 3.3"}],"stars":"","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":0.793,"topic":"Coreference reasoning (German)","total_models":1613,"url":""},{"bench_models":3,"description":"Weapons of Mass Destruction Proxy benchmark for biosecurity, measuring hazardous biological knowledge without info hazards.","github":"https://github.com/centerforaisafety/wmdp","has_recent":0,"id":"wmdp-bio","leaderboard":"","lower_is_better":true,"metric":"percent","name":"WMDP Bio","paper":"https://arxiv.org/abs/2403.03218","placements":[{"company":"huggingface","family":"Zephyr","model":"zephyr-7b","place":1,"score":63.7,"version":"Zephyr"},{"company":"xai","family":"Grok","model":"grok-4","place":2,"score":87,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":3,"score":87,"version":"Grok 4.1"}],"stars":"","top_company":"huggingface","top_model":"Zephyr 7B","top_model_family":"Zephyr","top_model_id":"zephyr-7b","top_model_version":"Zephyr","top_open_source":1,"top_percent":63.7,"topic":"Biosecurity knowledge","total_models":1613,"url":"https://www.wmdp.ai/"},{"bench_models":3,"description":"WMDP benchmark for chemical security, evaluating knowledge relevant to chemical weapons development.","github":"https://github.com/centerforaisafety/wmdp","has_recent":0,"id":"wmdp-chem","leaderboard":"","lower_is_better":true,"metric":"percent","name":"WMDP Chem","paper":"https://arxiv.org/abs/2403.03218","placements":[{"company":"huggingface","family":"Zephyr","model":"zephyr-7b","place":1,"score":45.8,"version":"Zephyr"},{"company":"xai","family":"Grok","model":"grok-4","place":2,"score":83,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":3,"score":84,"version":"Grok 4.1"}],"stars":"","top_company":"huggingface","top_model":"Zephyr 7B","top_model_family":"Zephyr","top_model_id":"zephyr-7b","top_model_version":"Zephyr","top_open_source":1,"top_percent":45.8,"topic":"Chemical security knowledge","total_models":1613,"url":"https://www.wmdp.ai/"},{"bench_models":3,"description":"WMDP benchmark for cybersecurity, assessing knowledge that could aid in cyber weapons development.","github":"https://github.com/centerforaisafety/wmdp","has_recent":0,"id":"wmdp-cyber","leaderboard":"","lower_is_better":true,"metric":"percent","name":"WMDP Cyber","paper":"https://arxiv.org/abs/2403.03218","placements":[{"company":"huggingface","family":"Zephyr","model":"zephyr-7b","place":1,"score":44,"version":"Zephyr"},{"company":"xai","family":"Grok","model":"grok-4","place":2,"score":79,"version":"Grok 4"},{"company":"xai","family":"Grok","model":"grok-4.1-thinking","place":3,"score":84,"version":"Grok 4.1"}],"stars":"","top_company":"huggingface","top_model":"Zephyr 7B","top_model_family":"Zephyr","top_model_id":"zephyr-7b","top_model_version":"Zephyr","top_open_source":1,"top_percent":44,"topic":"Cybersecurity knowledge","total_models":1613,"url":"https://www.wmdp.ai/"},{"bench_models":2,"description":"WMT16 English–German translation benchmark (news).","github":"","has_recent":0,"id":"wmt16","leaderboard":"","lower_is_better":false,"metric":"bleu","name":"WMT16 En–De","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":1,"score":38.841,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":2,"score":37.38,"version":"Llama TFree HAT"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B Instruct","top_model_family":"Llama","top_model_id":"llama-3.3-70b-instruct","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":38.841,"topic":"Machine translation","total_models":1613,"url":"https://www.statmt.org/wmt16/translation-task.html"},{"bench_models":2,"description":"Instruction-tuned evaluation on the WMT16 English–German translation set.","github":"","has_recent":0,"id":"wmt16-instruct","leaderboard":"","lower_is_better":false,"metric":"bleu","name":"WMT16 En–De (Instruct)","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":1,"score":37.912,"version":"Llama 3.3"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":2,"score":37.614,"version":"Llama TFree HAT"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B Instruct","top_model_family":"Llama","top_model_id":"llama-3.3-70b-instruct","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":37.912,"topic":"Machine translation","total_models":1613,"url":""},{"bench_models":33,"description":"Extended WMT 2024 evaluation across multiple language pairs.","github":"","has_recent":0,"id":"wmt24-plus-plus","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WMT24++","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":1,"score":94.7,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":2,"score":93.6,"version":"GPT-OSS"},{"company":"lg","family":"K-EXAONE","model":"k-exaone","place":3,"score":90.5,"version":"K-EXAONE"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-thinking","place":4,"score":90,"version":"DeepSeek V3.2"},{"company":"nvidia","family":"Nemotron","model":"nvidia-nemotron-3-nano-30b-a3b-bf16","place":5,"score":86.2,"version":"Nemotron 3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":6,"score":85.6,"version":"Qwen3"},{"company":"google","family":"Gemma","model":"gemma-3-27b","place":7,"score":83.65,"version":"Gemma 3"},{"company":"eurollm","family":"EuroLLM","model":"eurollm-22b","place":8,"score":83.62,"version":"EuroLLM"},{"company":"openai","family":"GPT","model":"gpt-oss-20b","place":9,"score":83.2,"version":"GPT-OSS"},{"company":"meta","family":"Llama","model":"llama-3.3-70b","place":10,"score":82.28,"version":"Llama 3.3"},{"company":"swissai","family":"Apertus","model":"apertus-70b","place":11,"score":81.96,"version":"Apertus"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":12,"score":81.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b","place":13,"score":81.69,"version":"Qwen3"},{"company":"lg","family":"EXAONE","model":"exaone-4.0-32b-thinking","place":14,"score":80.8,"version":"EXAONE 4.0"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":15,"score":80.7,"version":"Gemini 3"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":16,"score":80.7,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":17,"score":79.7,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":18,"score":78.9,"version":"Qwen3.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":19,"score":78.8,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":20,"score":78.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-max-thinking","place":21,"score":77.6,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":22,"score":77.6,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":23,"score":77.6,"version":"Qwen3.5"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":24,"score":76.38,"version":"OLMo 2"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":25,"score":76.3,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":26,"score":75.8,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-oss-120b","place":27,"score":74.4,"version":"GPT-OSS"},{"company":"mistral","family":"Mistral","model":"mistral-3.2-24b","place":28,"score":72.81,"version":"Mistral 3.2"},{"company":"google","family":"Gemma","model":"gemma-3-it-27b","place":29,"score":53.4,"version":"Gemma 3"},{"company":"cohere","family":"Aya","model":"tiny-aya-global","place":30,"score":45.4,"version":"Aya"},{"company":"google","family":"Gemma","model":"gemma-3-4b","place":31,"score":40.5,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":32,"score":32.6,"version":"Qwen3"},{"company":"mistral","family":"Ministral","model":"ministral-3-3b-base","place":33,"score":30.4,"version":"Ministral 3"}],"stars":"","top_company":"qwen","top_model":"Qwen3-235B-A22B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-235b-a22b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":94.7,"topic":"Machine translation","total_models":1613,"url":"https://arxiv.org/abs/2502.12404v1"},{"bench_models":5,"description":"WorldTravel2 benchmark multimodal track.","github":"","has_recent":0,"id":"worldtravel2-multimodal","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WorldTravel2 (multi-modal)","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":47.2,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":47.2,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":45.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":41.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":36,"version":"Gemini 2.5"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":47.2,"topic":"Travel planning (multimodal)","total_models":1613,"url":""},{"bench_models":5,"description":"WorldTravel2 benchmark text-only track.","github":"","has_recent":0,"id":"worldtravel2-text","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WorldTravel2 (text)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":56.4,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":2,"score":53.3,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":53.3,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":52.1,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":44.5,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":56.4,"topic":"Travel planning (text)","total_models":1613,"url":""},{"bench_models":5,"description":"Visual question answering requiring world knowledge and commonsense reasoning.","github":"","has_recent":0,"id":"worldvqa","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WorldVQA","paper":"","placements":[{"company":"google","family":"Gemini","model":"gemini-3-pro","place":1,"score":47.4,"version":"Gemini 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":2,"score":46.3,"version":"Kimi K2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":3,"score":36.8,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":4,"score":28,"version":"GPT-5.2"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":5,"score":23.5,"version":"Qwen3 VL"}],"stars":"","top_company":"google","top_model":"Gemini 3 Pro","top_model_family":"Gemini","top_model_id":"gemini-3-pro","top_model_version":"Gemini 3","top_open_source":0,"top_percent":47.4,"topic":"World knowledge VQA","total_models":1613,"url":""},{"bench_models":27,"description":"General-purpose writing quality benchmark.","github":"","has_recent":0,"id":"writingbench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WritingBench","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b-thinking-2507","place":1,"score":88.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-instruct","place":2,"score":87.3,"version":"Qwen3 Next"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-thinking","place":3,"score":85.9,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":4,"score":85.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":5,"score":85.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-thinking","place":6,"score":85.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":7,"score":85.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-thinking","place":8,"score":85.2,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":9,"score":85,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-next-80b-a3b-thinking","place":10,"score":84.6,"version":"Qwen3 Next"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-thinking","place":11,"score":83.9,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-4b-instruct-2507","place":12,"score":83.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":13,"score":83.1,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":14,"score":83,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":15,"score":82.6,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":16,"score":82.6,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":17,"score":82.5,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-thinking","place":18,"score":80.3,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":19,"score":79,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":20,"score":77,"version":"Qwen3"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":21,"score":75.5,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":22,"score":75.4,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":23,"score":71.4,"version":"Qwen3"},{"company":"upstage","family":"Solar","model":"solar-open-100b","place":24,"score":7.51,"version":"Solar Open"},{"company":"zai","family":"GLM","model":"glm-4.5-air","place":25,"score":7.4,"version":"GLM 4.5"},{"company":"openai","family":"GPT","model":"gpt-oss-120b-high","place":26,"score":6.61,"version":"GPT-OSS"},{"company":"gpt-oss","family":"GPT-OSS","model":"gpt-oss-120b-medium","place":27,"score":6.55,"version":"GPT-OSS"}],"stars":"","top_company":"qwen","top_model":"Qwen3-235B-A22B-Thinking-2507","top_model_family":"Qwen","top_model_id":"qwen3-235b-a22b-thinking-2507","top_model_version":"Qwen3","top_open_source":1,"top_percent":88.3,"topic":"Writing quality","total_models":1613,"url":""},{"bench_models":14,"description":"Classic Winograd Schema Challenge measuring commonsense coreference.","github":"","has_recent":0,"id":"wsc","leaderboard":"","lower_is_better":false,"metric":"percent","name":"WSC","paper":"https://arxiv.org/abs/1109.1334","placements":[{"company":"google","family":"Gemma","model":"gemma-3-pt-27b","place":1,"score":91.9,"version":"Gemma 3"},{"company":"allenai","family":"OLMo","model":"olmo-2-32b-base","place":2,"score":86.8,"version":"OLMo 2"},{"company":"nvidia","family":"Nemotron","model":"nemotron-nano-12b-v2-base","place":3,"score":85.4,"version":"Nemotron Nano v2"},{"company":"marin","family":"Marin","model":"marin-32b-bison","place":4,"score":82.4,"version":"Marin 32B"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-base","place":5,"score":81,"version":"Qwen2.5"},{"company":"marin","family":"Marin","model":"marin-32b-mantis","place":6,"score":79.5,"version":"Marin 32B"},{"company":"eleutherai","family":"Pythia","model":"pythia-410m","place":7,"score":47.1,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1.4b","place":8,"score":44.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-6.9b","place":9,"score":44.2,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-12b","place":10,"score":39.4,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-2.8b","place":11,"score":38.5,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-70m","place":12,"score":36.5,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-160m","place":13,"score":36.5,"version":"Pythia"},{"company":"eleutherai","family":"Pythia","model":"pythia-1b","place":14,"score":36.5,"version":"Pythia"}],"stars":"","top_company":"google","top_model":"Gemma 3 PT 27B","top_model_family":"Gemma","top_model_id":"gemma-3-pt-27b","top_model_version":"Gemma 3","top_open_source":1,"top_percent":91.9,"topic":"Coreference reasoning","total_models":1613,"url":"https://cs.nyu.edu/faculty/davise/papers/WinogradSchemas/WS.html"},{"bench_models":22,"description":"Evaluates multi-hop deep research workflows on xBench DeepSearch tasks.","github":"","has_recent":0,"id":"xbench-deepsearch","leaderboard":"","lower_is_better":false,"metric":"avg@3","name":"xBench-DeepSearch","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":77.9,"version":"GPT-5"},{"company":"openai","family":"GPT","model":"gpt-5-thinking","place":2,"score":77.8,"version":"GPT-5"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.0-72b","place":3,"score":77.8,"version":"MiroThinker v1.0"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-235b","place":4,"score":77.1,"version":"MiroThinker v1.5"},{"company":"tongyi","family":"Tongyi","model":"tongyi-deepresearch","place":5,"score":75,"version":"Tongyi"},{"company":"miromind","family":"MiroThinker","model":"mirothinker-v1.5-30b","place":6,"score":73.1,"version":"MiroThinker v1.5"},{"company":"minimax","family":"MiniMax","model":"minimax-m2","place":7,"score":72,"version":"MiniMax M2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.1","place":8,"score":71,"version":"DeepSeek V3.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2-exp","place":9,"score":71,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.5","place":10,"score":70,"version":"GLM 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6","place":11,"score":70,"version":"GLM 4.6"},{"company":"moonshotai","family":"Kimi","model":"kimi-researcher","place":12,"score":69,"version":"Kimi Researcher"},{"company":"openai","family":"o3","model":"o3","place":13,"score":67,"version":"o3"},{"company":"anthropic","family":"Claude","model":"claude-4.5-sonnet","place":14,"score":66,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":15,"score":65,"version":"Claude 4"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":16,"score":64.6,"version":"Claude 4"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-0905-preview","place":17,"score":61,"version":"Kimi K2"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":18,"score":56,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2","place":19,"score":50,"version":"Kimi K2"},{"company":"tencent","family":"Youtu-LLM","model":"youtu-llm-2b","place":20,"score":19.5,"version":"Youtu-LLM"},{"company":"qwen","family":"Qwen","model":"qwen3-4b","place":21,"score":18.4,"version":"Qwen3"},{"company":"huggingface","family":"SmolLM","model":"smollm3-3b","place":22,"score":13.9,"version":"SmollM3"},{"company":"qwen","family":"Qwen","model":"qwen3-1.7b","place":23,"score":11.7,"version":"Qwen3"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":77.9,"topic":"Agentic research","total_models":1613,"url":""},{"bench_models":7,"description":"xBench DeepSearch benchmark May 2025 snapshot.","github":"","has_recent":0,"id":"xbench-deepsearch-2025.05","leaderboard":"","lower_is_better":false,"metric":"avg@3","name":"xBench-DeepSearch (2025.05)","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":1,"score":83.7,"version":"Step 3.5 Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":2,"score":78,"version":"DeepSeek V3.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":3,"score":76.7,"version":"Kimi K2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-thinking","place":4,"score":76,"version":"Kimi K2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":5,"score":72,"version":"GLM 4.7"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":6,"score":69.3,"version":"MiMo V2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":7,"score":68.7,"version":"MiniMax M2.1"}],"stars":"","top_company":"stepfun","top_model":"Step-3.5 Flash 20260204","top_model_family":"Step","top_model_id":"step-3.5-flash-20260204","top_model_version":"Step 3.5 Flash","top_open_source":1,"top_percent":83.7,"topic":"Agentic research","total_models":1613,"url":""},{"bench_models":6,"description":"xBench DeepSearch benchmark October 2025 snapshot.","github":"","has_recent":0,"id":"xbench-deepsearch-2025.10","leaderboard":"","lower_is_better":false,"metric":"avg@3","name":"xBench-DeepSearch (2025.10)","paper":"","placements":[{"company":"stepfun","family":"Step","model":"step-3.5-flash-20260204","place":1,"score":56.3,"version":"Step 3.5 Flash"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3.2","place":2,"score":55.7,"version":"DeepSeek V3.2"},{"company":"zai","family":"GLM","model":"glm-4.7","place":3,"score":52.3,"version":"GLM 4.7"},{"company":"xiaomi","family":"MiMo","model":"mimo-v2-flash","place":4,"score":44,"version":"MiMo V2"},{"company":"minimax","family":"MiniMax","model":"minimax-m2.1","place":5,"score":43,"version":"MiniMax M2.1"},{"company":"boss","family":"Nanbeige","model":"nanbeige4.1-3b","place":6,"score":39,"version":"Nanbeige4.1"}],"stars":"","top_company":"stepfun","top_model":"Step-3.5 Flash 20260204","top_model_family":"Step","top_model_id":"step-3.5-flash-20260204","top_model_version":"Step 3.5 Flash","top_open_source":1,"top_percent":56.3,"topic":"Agentic research","total_models":1613,"url":""},{"bench_models":5,"description":"Remote sensing benchmark for evaluating multimodal AI on satellite and aerial imagery.","github":"","has_recent":0,"id":"xlrs-bench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"XLRS-Bench","paper":"","placements":[{"company":"internlm","family":"Intern","model":"intern-s1-pro","place":1,"score":52.8,"version":"Intern-S1"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":51.8,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":3,"score":51.2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":4,"score":50.4,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":5,"score":46.4,"version":"Kimi K2.5"}],"stars":"","top_company":"internlm","top_model":"Intern-S1-Pro","top_model_family":"Intern","top_model_id":"intern-s1-pro","top_model_version":"Intern-S1","top_open_source":1,"top_percent":52.8,"topic":"Remote sensing","total_models":1613,"url":""},{"bench_models":5,"description":"XpertBench education domain subset.","github":"","has_recent":0,"id":"xpertbench-edu","leaderboard":"","lower_is_better":false,"metric":"percent","name":"XpertBench (Edu)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":56.9,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":49.2,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":3,"score":47.9,"version":"Gemini 2.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":4,"score":47.9,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":5,"score":44.5,"version":"Claude 4.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":56.9,"topic":"Economics/education","total_models":1613,"url":""},{"bench_models":5,"description":"XpertBench finance domain subset.","github":"","has_recent":0,"id":"xpertbench-fin","leaderboard":"","lower_is_better":false,"metric":"percent","name":"XpertBench (Fin)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":64.5,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":62,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":3,"score":56.1,"version":"Gemini 3"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":44.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":30.3,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":64.5,"topic":"Economics/finance","total_models":1613,"url":""},{"bench_models":5,"description":"XpertBench humanities domain subset.","github":"","has_recent":0,"id":"xpertbench-humanities","leaderboard":"","lower_is_better":false,"metric":"percent","name":"XpertBench (Humanities)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":68.5,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":68.2,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":60.2,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":54.9,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":52.3,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":68.5,"topic":"Economics/humanities","total_models":1613,"url":""},{"bench_models":5,"description":"XpertBench legal domain subset.","github":"","has_recent":0,"id":"xpertbench-law","leaderboard":"","lower_is_better":false,"metric":"percent","name":"XpertBench (Law)","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":1,"score":58.7,"version":"Claude 4.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":2,"score":55.2,"version":"Seed 1.8"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":3,"score":54.7,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":4,"score":52.3,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":47.3,"version":"Gemini 2.5"}],"stars":"","top_company":"anthropic","top_model":"Claude Sonnet 4.5","top_model_family":"Claude","top_model_id":"claude-sonnet-4.5","top_model_version":"Claude 4.5","top_open_source":0,"top_percent":58.7,"topic":"Economics/legal","total_models":1613,"url":""},{"bench_models":5,"description":"XpertBench research domain subset.","github":"","has_recent":0,"id":"xpertbench-research","leaderboard":"","lower_is_better":false,"metric":"percent","name":"XpertBench (Research)","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-5-high","place":1,"score":48.2,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":34.9,"version":"Gemini 3"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":3,"score":31.4,"version":"Seed 1.8"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":4,"score":27.5,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":5,"score":25.5,"version":"Gemini 2.5"}],"stars":"","top_company":"openai","top_model":"GPT-5 High","top_model_family":"GPT","top_model_id":"gpt-5-high","top_model_version":"GPT-5","top_open_source":0,"top_percent":48.2,"topic":"Economics/research","total_models":1613,"url":""},{"bench_models":4,"description":"XSTest safety benchmark.","github":"","has_recent":0,"id":"xstest","leaderboard":"","lower_is_better":false,"metric":"percent","name":"XSTest","paper":"","placements":[{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-thinking","place":1,"score":94.3,"version":"IQuest Coder V1"},{"company":"qwen","family":"Qwen","model":"qwen2.5-coder-32b-instruct","place":2,"score":90.6,"version":"Qwen2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-coder-480b-a35b-instruct","place":3,"score":90.1,"version":"Qwen3"},{"company":"iquest","family":"IQuest Coder","model":"iquest-coder-v1-40b-instruct","place":4,"score":89.3,"version":"IQuest Coder V1"}],"stars":"","top_company":"iquest","top_model":"IQuest-Coder-V1-40B-Thinking","top_model_family":"IQuest Coder","top_model_id":"iquest-coder-v1-40b-thinking","top_model_version":"IQuest Coder V1","top_open_source":1,"top_percent":94.3,"topic":"Safety","total_models":1613,"url":""},{"bench_models":53,"description":"Logical reasoning benchmark assessing complex pattern and rule inference.","github":"","has_recent":0,"id":"zebralogic","leaderboard":"https://huggingface.co/spaces/allenai/ZeroEval","lower_is_better":false,"metric":"percent","name":"ZebraLogic","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-thinking","place":1,"score":96.1,"version":"Qwen3 VL"},{"company":"openai","family":"o3","model":"o3","place":2,"score":95.8,"version":"o3"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-0528","place":3,"score":95.1,"version":"DeepSeek R1"},{"company":"anthropic","family":"Claude","model":"claude-4-opus","place":4,"score":95.1,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-moe-2507","place":5,"score":94.2,"version":"Qwen3"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":91.6,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":7,"score":90.97,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-instruct-2507","place":8,"score":90,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b-thinking-2507","place":9,"score":89.5,"version":"Qwen3"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2-instruct","place":10,"score":89,"version":"Kimi K2"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-q","place":11,"score":88.9,"version":"LLaDA 2.1"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-thinking","place":12,"score":88.8,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b-thinking","place":13,"score":88.5,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-32b","place":14,"score":88.3,"version":"Qwen3"},{"company":"antgroup","family":"Ling","model":"ling-flash-2.0","place":15,"score":87.6,"version":"Ling 2.0"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-80k","place":16,"score":86.8,"version":"MiniMax M1"},{"company":"bytedance","family":"Seed","model":"seed-thinking-v1.5","place":17,"score":84.4,"version":"Seed 1.5"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-flash-s","place":18,"score":84.2,"version":"LLaDA 2.1"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3-0324","place":19,"score":84,"version":"DeepSeek V3"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-flash","place":20,"score":82.3,"version":"LLaDA 2.0"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-v3","place":21,"score":81.8,"version":"DeepSeek V3"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":22,"score":80.3,"version":"Qwen3"},{"company":"minimax","family":"MiniMax","model":"minimax-m1-40k","place":23,"score":80.1,"version":"MiniMax M1"},{"company":"allenai","family":"OLMo","model":"olmo-3.1-think-32b","place":24,"score":80.1,"version":"OLMo 3.1"},{"company":"antgroup","family":"Ling","model":"ling-mini-2.0","place":25,"score":79.85,"version":"Ling 2.0"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":26,"score":79.7,"version":"Claude 4"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1","place":27,"score":78.7,"version":"DeepSeek R1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-q","place":28,"score":77.1,"version":"LLaDA 2.1"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-reasoning","place":29,"score":77,"version":"Motif 2"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-flash-instruct","place":30,"score":76.1,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-omni-30b-a3b-instruct","place":31,"score":76,"version":"Qwen3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b","place":32,"score":76,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-dpo","place":33,"score":74.5,"version":"OLMo 3"},{"company":"allenai","family":"OLMo","model":"olmo-3-think-32b-sft","place":34,"score":70.5,"version":"OLMo 3"},{"company":"motif","family":"Motif","model":"motif-2-12.7b-instruct","place":35,"score":69.5,"version":"Motif 2"},{"company":"deepseek","family":"DeepSeek","model":"deepseek-r1-distill-qwen-32b","place":36,"score":69.4,"version":"DeepSeek R1"},{"company":"antgroup","family":"LLaDA","model":"llada2.1-mini-s","place":37,"score":68.5,"version":"LLaDA 2.1"},{"company":"antgroup","family":"LLaDA","model":"llada2.0-mini","place":38,"score":64.2,"version":"LLaDA 2.0"},{"company":"anthropic","family":"Claude","model":"claude-opus-4","place":39,"score":59.3,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":40,"score":58.5,"version":"GPT-4.1"},{"company":"baidu","family":"ERNIE","model":"ernie-4.5","place":41,"score":58.1,"version":"ERNIE 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash","place":42,"score":57.9,"version":"Gemini 2.5"},{"company":"openai","family":"GPT","model":"gpt-4.1","place":43,"score":56.3,"version":"GPT-4.1"},{"company":"openai","family":"GPT","model":"gpt-4o-0327","place":44,"score":52.6,"version":"GPT-4o"},{"company":"qwen","family":"Qwen","model":"qwen3-235b-a22b","place":45,"score":37.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen-3-235b-a22b-nonthinking","place":46,"score":37.7,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-8b-non-thinking","place":47,"score":37.48,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-30b-a3b","place":48,"score":33.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-14b","place":49,"score":33,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen3-32b-nothinking","place":50,"score":29.2,"version":"Qwen3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-72b-instruct","place":51,"score":26.6,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-3-27b-it","place":52,"score":24.8,"version":"Gemma 3"},{"company":"qwen","family":"Qwen","model":"qwen2.5-32b-instruct","place":53,"score":24.1,"version":"Qwen2.5"},{"company":"google","family":"Gemma","model":"gemma-2-27b-it","place":54,"score":17.2,"version":"Gemma 2"},{"company":"allenai","family":"OLMo","model":"olmo-2-0325-32b-instruct","place":55,"score":13.3,"version":"OLMo 2"},{"company":"swissai","family":"Apertus","model":"apertus-70b-instruct","place":56,"score":9,"version":"Apertus"}],"stars":"","top_company":"qwen","top_model":"Qwen3-VL 32B Thinking","top_model_family":"Qwen","top_model_id":"qwen3-vl-32b-thinking","top_model_version":"Qwen3 VL","top_open_source":1,"top_percent":96.1,"topic":"Logical reasoning","total_models":1613,"url":""},{"bench_models":28,"description":"Evaluates zero-shot performance across diverse tasks without task-specific finetuning.","github":"","has_recent":0,"id":"zerobench","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ZeroBench","paper":"","placements":[{"company":"zai","family":"GLM","model":"glm-4.5v","place":1,"score":23.4,"version":"GLM 4.5V"},{"company":"stepfun","family":"Step","model":"step-3","place":2,"score":23,"version":"Step 3"},{"company":"zai","family":"GLM","model":"glm-4.1v","place":3,"score":19.2,"version":"GLM 4.1V"},{"company":"google","family":"Gemma","model":"gemma-3-27b-vl","place":4,"score":17.7,"version":"Gemma 3"},{"company":"moonshotai","family":"Kimi","model":"kimi-vl-a3b-thinking-2506","place":5,"score":16.2,"version":"Kimi VL"},{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":6,"score":12,"version":"Qwen3.5"},{"company":"bytedance","family":"Seed","model":"seed1.8","place":7,"score":11,"version":"Seed 1.8"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":8,"score":10,"version":"Gemini 3"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":9,"score":10,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":10,"score":10,"version":"Qwen3.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5-thinking","place":11,"score":9,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2-thinking-xhigh","place":12,"score":9,"version":"GPT-5.2"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":13,"score":9,"version":"GPT-5.2"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":14,"score":9,"version":"Kimi K2.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":15,"score":9,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":16,"score":8,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":17,"score":8,"version":"Qwen3.5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":18,"score":6,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":19,"score":5,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":20,"score":4,"version":"Qwen3 VL"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":21,"score":4,"version":"dots.vlm1"},{"company":"openai","family":"GPT","model":"gpt-5-high","place":22,"score":4,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":23,"score":4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":24,"score":4,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":25,"score":4,"version":"Claude 4.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":26,"score":3,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5-thinking","place":27,"score":3,"version":"Claude 4.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":28,"score":3,"version":"Claude 4.5"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":29,"score":3,"version":"GPT-5"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":30,"score":2,"version":"Seed 1.5"},{"company":"openai","family":"GPT","model":"gpt-5","place":31,"score":2,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":32,"score":2,"version":"Qwen2.5 VL"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":33,"score":1,"version":"Claude 4.1"}],"stars":"","top_company":"zai","top_model":"GLM-4.5V","top_model_family":"GLM","top_model_id":"glm-4.5v","top_model_version":"GLM 4.5V","top_open_source":1,"top_percent":23.4,"topic":"Zero-shot generalization","total_models":1613,"url":""},{"bench_models":31,"description":"Subset of ZeroBench targeting harder zero-shot reasoning cases.","github":"","has_recent":0,"id":"zerobench-sub","leaderboard":"","lower_is_better":false,"metric":"percent","name":"ZeroBench (sub)","paper":"","placements":[{"company":"qwen","family":"Qwen","model":"qwen3.5-397b-a17b","place":1,"score":41,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-3-pro","place":2,"score":39,"version":"Gemini 3"},{"company":"qwen","family":"Qwen","model":"qwen3.5-122b-a10b","place":3,"score":36.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-27b","place":4,"score":36.2,"version":"Qwen3.5"},{"company":"qwen","family":"Qwen","model":"qwen3.5-35b-a3b","place":5,"score":34.1,"version":"Qwen3.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":6,"score":33.8,"version":"Gemini 2.5"},{"company":"moonshotai","family":"Kimi","model":"kimi-k2.5","place":7,"score":33.5,"version":"Kimi K2.5"},{"company":"openai","family":"GPT","model":"gpt-5.2","place":8,"score":33.2,"version":"GPT-5.2"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":9,"score":30.8,"version":"Seed 1.5"},{"company":"google","family":"Gemini","model":"gemini-2.5-pro","place":10,"score":30.24,"version":"Gemini 2.5"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.5","place":11,"score":28.4,"version":"Claude 4.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b","place":12,"score":28.4,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-235b-a22b-thinking","place":13,"score":27.7,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini","place":14,"score":27.3,"version":"GPT-5"},{"company":"rednote","family":"dots","model":"dots-vlm1","place":15,"score":26.65,"version":"dots.vlm1"},{"company":"anthropic","family":"Claude","model":"claude-opus-4.1","place":16,"score":26.3,"version":"Claude 4.1"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4.5","place":17,"score":26.3,"version":"Claude 4.5"},{"company":"zai","family":"GLM","model":"glm-4.6v","place":18,"score":25.8,"version":"GLM 4.6V"},{"company":"bytedance","family":"Seed","model":"seed1.5-vl-thinking","place":19,"score":25.75,"version":"Seed 1.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-32b-instruct","place":20,"score":25.2,"version":"Qwen3 VL"},{"company":"openai","family":"GPT","model":"gpt-5-mini-minimal","place":21,"score":24.9,"version":"GPT-5"},{"company":"anthropic","family":"Claude","model":"claude-sonnet-4","place":22,"score":24.9,"version":"Claude 4"},{"company":"openai","family":"GPT","model":"gpt-5","place":23,"score":24.6,"version":"GPT-5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-30b-a3b-instruct","place":24,"score":24,"version":"Qwen3 VL"},{"company":"anthropic","family":"Claude","model":"claude-4-sonnet","place":25,"score":23.1,"version":"Claude 4"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-8b-instruct","place":26,"score":22.8,"version":"Qwen3 VL"},{"company":"zai","family":"GLM","model":"glm-4.6v-flash","place":27,"score":22.5,"version":"GLM 4.6V"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-4b-instruct","place":28,"score":21,"version":"Qwen3 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b","place":29,"score":20,"version":"Qwen2.5 VL"},{"company":"qwen","family":"Qwen","model":"qwen2.5-vl-72b-instruct","place":30,"score":18,"version":"Qwen2.5 VL"},{"company":"openai","family":"GPT","model":"gpt-5-nano-minimal","place":31,"score":15.9,"version":"GPT-5"},{"company":"google","family":"Gemini","model":"gemini-2.5-flash-lite","place":32,"score":15.3,"version":"Gemini 2.5"},{"company":"qwen","family":"Qwen","model":"qwen3-vl-2b-instruct","place":33,"score":13.2,"version":"Qwen3 VL"}],"stars":"","top_company":"qwen","top_model":"Qwen3.5-397B-A17B","top_model_family":"Qwen","top_model_id":"qwen3.5-397b-a17b","top_model_version":"Qwen3.5","top_open_source":1,"top_percent":41,"topic":"Zero-shot generalization","total_models":1613,"url":""},{"bench_models":7,"description":"ZeroSCROLLS split based on BookSumSort long-form summarization.","github":"","has_recent":0,"id":"zeroscrolls-booksumsort","leaderboard":"","lower_is_better":false,"metric":"cidx","name":"ZeroSCROLLS BookSumSort","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-4","place":1,"score":60.5,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":2,"score":49.8,"version":"GPT-3.5"},{"company":"openai","family":"Davinci","model":"davinci-003","place":3,"score":49.5,"version":"Davinci"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":4,"score":47.4,"version":"Claude v1.3"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":5,"score":16.4,"version":"FLAN-T5"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":6,"score":14,"version":"FLAN-UL2"},{"company":"bigscience","family":"T0","model":"t0pp","place":7,"score":0,"version":"T0pp"}],"stars":"","top_company":"openai","top_model":"GPT-4","top_model_family":"GPT","top_model_id":"gpt-4","top_model_version":"GPT-4","top_open_source":0,"top_percent":60.5,"topic":"Long-context summarization","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":8,"description":"ZeroSCROLLS split based on the GovReport summarization benchmark.","github":"","has_recent":0,"id":"zeroscrolls-govreport","leaderboard":"","lower_is_better":false,"metric":"rgeo","name":"ZeroSCROLLS GovReport","paper":"","placements":[{"company":"google","family":"CoLT5","model":"colt5","place":1,"score":41,"version":"CoLT5"},{"company":"openai","family":"GPT","model":"gpt-4","place":2,"score":26.3,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":3,"score":24.2,"version":"Claude v1.3"},{"company":"openai","family":"Davinci","model":"davinci-003","place":4,"score":21.7,"version":"Davinci"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":5,"score":21.3,"version":"GPT-3.5"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":6,"score":17.6,"version":"FLAN-T5"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":7,"score":16.1,"version":"FLAN-UL2"},{"company":"bigscience","family":"T0","model":"t0pp","place":8,"score":7.1,"version":"T0pp"}],"stars":"","top_company":"google","top_model":"CoLT5","top_model_family":"CoLT5","top_model_id":"colt5","top_model_version":"CoLT5","top_open_source":1,"top_percent":41,"topic":"Long-context summarization","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":9,"description":"ZeroSCROLLS split derived from MuSiQue multi-hop QA.","github":"","has_recent":0,"id":"zeroscrolls-musique","leaderboard":"","lower_is_better":false,"metric":"f1","name":"ZeroSCROLLS MuSiQue","paper":"","placements":[{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":1,"score":52.2,"version":"Llama 3.3"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":2,"score":51.3,"version":"FLAN-UL2"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":3,"score":46.8,"version":"FLAN-T5"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":4,"score":45,"version":"Llama TFree HAT"},{"company":"openai","family":"GPT","model":"gpt-4","place":5,"score":41.1,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":6,"score":36.1,"version":"Claude v1.3"},{"company":"bigscience","family":"T0","model":"t0pp","place":7,"score":35.3,"version":"T0pp"},{"company":"openai","family":"Davinci","model":"davinci-003","place":8,"score":33.5,"version":"Davinci"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":9,"score":27.1,"version":"GPT-3.5"}],"stars":"","top_company":"meta","top_model":"Llama 3.3 70B Instruct","top_model_family":"Llama","top_model_id":"llama-3.3-70b-instruct","top_model_version":"Llama 3.3","top_open_source":1,"top_percent":52.2,"topic":"Long-context reasoning","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":8,"description":"ZeroSCROLLS split based on the NarrativeQA reading comprehension benchmark.","github":"","has_recent":0,"id":"zeroscrolls-narrativeqa","leaderboard":"","lower_is_better":false,"metric":"f1","name":"ZeroSCROLLS NarrativeQA","paper":"","placements":[{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":1,"score":32.6,"version":"Claude v1.3"},{"company":"google","family":"CoLT5","model":"colt5","place":2,"score":31,"version":"CoLT5"},{"company":"openai","family":"GPT","model":"gpt-4","place":3,"score":27.6,"version":"GPT-4"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":4,"score":25.5,"version":"FLAN-UL2"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":5,"score":25.1,"version":"GPT-3.5"},{"company":"openai","family":"Davinci","model":"davinci-003","place":6,"score":24.6,"version":"Davinci"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":7,"score":19.3,"version":"FLAN-T5"},{"company":"bigscience","family":"T0","model":"t0pp","place":8,"score":18.7,"version":"T0pp"}],"stars":"","top_company":"anthropic","top_model":"Claude v1.3","top_model_family":"Claude","top_model_id":"claude-v1.3","top_model_version":"Claude v1.3","top_open_source":0,"top_percent":32.6,"topic":"Long-context QA","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":8,"description":"ZeroSCROLLS split based on the Qasper paper QA benchmark.","github":"","has_recent":0,"id":"zeroscrolls-qasper","leaderboard":"","lower_is_better":false,"metric":"f1","name":"ZeroSCROLLS Qasper","paper":"","placements":[{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":1,"score":56.9,"version":"FLAN-UL2"},{"company":"google","family":"CoLT5","model":"colt5","place":2,"score":53.1,"version":"CoLT5"},{"company":"openai","family":"Davinci","model":"davinci-003","place":3,"score":52.7,"version":"Davinci"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":4,"score":52.3,"version":"Claude v1.3"},{"company":"openai","family":"GPT","model":"gpt-4","place":5,"score":50.7,"version":"GPT-4"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":6,"score":49.3,"version":"GPT-3.5"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":7,"score":48.3,"version":"FLAN-T5"},{"company":"bigscience","family":"T0","model":"t0pp","place":8,"score":25,"version":"T0pp"}],"stars":"","top_company":"google","top_model":"FLAN-UL2","top_model_family":"FLAN-UL2","top_model_id":"flan-ul2","top_model_version":"FLAN-UL2","top_open_source":1,"top_percent":56.9,"topic":"Long-context QA","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":8,"description":"ZeroSCROLLS split based on the QMSum meeting summarization benchmark.","github":"","has_recent":0,"id":"zeroscrolls-qmsum","leaderboard":"","lower_is_better":false,"metric":"rgeo","name":"ZeroSCROLLS QMSum","paper":"","placements":[{"company":"google","family":"CoLT5","model":"colt5","place":1,"score":22.5,"version":"CoLT5"},{"company":"openai","family":"GPT","model":"gpt-4","place":2,"score":18.5,"version":"GPT-4"},{"company":"openai","family":"Davinci","model":"davinci-003","place":3,"score":16.9,"version":"Davinci"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":4,"score":15.6,"version":"GPT-3.5"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":5,"score":14.6,"version":"Claude v1.3"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":6,"score":13.6,"version":"FLAN-UL2"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":7,"score":11,"version":"FLAN-T5"},{"company":"bigscience","family":"T0","model":"t0pp","place":8,"score":7.2,"version":"T0pp"}],"stars":"","top_company":"google","top_model":"CoLT5","top_model_family":"CoLT5","top_model_id":"colt5","top_model_version":"CoLT5","top_open_source":1,"top_percent":22.5,"topic":"Long-context summarization","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":8,"description":"ZeroSCROLLS split based on the QuALITY reading comprehension benchmark.","github":"","has_recent":0,"id":"zeroscrolls-quality","leaderboard":"","lower_is_better":false,"metric":"ac","name":"ZeroSCROLLS QuALITY","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-4","place":1,"score":89.2,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":2,"score":84.8,"version":"Claude v1.3"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":3,"score":75.6,"version":"FLAN-UL2"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":4,"score":75.2,"version":"FLAN-T5"},{"company":"openai","family":"Davinci","model":"davinci-003","place":5,"score":69,"version":"Davinci"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":6,"score":66.6,"version":"GPT-3.5"},{"company":"google","family":"CoLT5","model":"colt5","place":7,"score":47,"version":"CoLT5"},{"company":"bigscience","family":"T0","model":"t0pp","place":8,"score":21.4,"version":"T0pp"}],"stars":"","top_company":"openai","top_model":"GPT-4","top_model_family":"GPT","top_model_id":"gpt-4","top_model_version":"GPT-4","top_open_source":0,"top_percent":89.2,"topic":"Long-context QA","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":9,"description":"ZeroSCROLLS SpaceDigest extractive summarization task.","github":"","has_recent":0,"id":"zeroscrolls-spacedigest","leaderboard":"","lower_is_better":false,"metric":"es","name":"ZeroSCROLLS SpaceDigest","paper":"","placements":[{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":1,"score":77.9,"version":"Llama TFree HAT"},{"company":"openai","family":"GPT","model":"gpt-4","place":2,"score":62.8,"version":"GPT-4"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":3,"score":61.6,"version":"Claude v1.3"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":4,"score":49.1,"version":"GPT-3.5"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":5,"score":48.7,"version":"FLAN-T5"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":6,"score":40.4,"version":"Llama 3.3"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":7,"score":36,"version":"FLAN-UL2"},{"company":"openai","family":"Davinci","model":"davinci-003","place":8,"score":31.3,"version":"Davinci"},{"company":"bigscience","family":"T0","model":"t0pp","place":9,"score":15.2,"version":"T0pp"}],"stars":"","top_company":"alephalpha","top_model":"Llama-3_1-70B-TFree-HAT-SFT","top_model_family":"Llama TFree HAT","top_model_id":"llama-3_1-70b-tfree-hat-sft","top_model_version":"Llama TFree HAT","top_open_source":1,"top_percent":77.9,"topic":"Long-context summarization","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":9,"description":"ZeroSCROLLS split based on the SQuALITY long-form summarization benchmark.","github":"","has_recent":0,"id":"zeroscrolls-squality","leaderboard":"","lower_is_better":false,"metric":"rouge_gm","name":"ZeroSCROLLS SQuALITY","paper":"","placements":[{"company":"openai","family":"GPT","model":"gpt-4","place":1,"score":22.6,"version":"GPT-4"},{"company":"openai","family":"Davinci","model":"davinci-003","place":2,"score":22,"version":"Davinci"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":3,"score":21,"version":"Claude v1.3"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":4,"score":20.4,"version":"GPT-3.5"},{"company":"alephalpha","family":"Llama TFree HAT","model":"llama-3_1-70b-tfree-hat-sft","place":5,"score":17,"version":"Llama TFree HAT"},{"company":"meta","family":"Llama","model":"llama-3.3-70b-instruct","place":6,"score":15.9,"version":"Llama 3.3"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":7,"score":8,"version":"FLAN-T5"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":8,"score":5.7,"version":"FLAN-UL2"},{"company":"bigscience","family":"T0","model":"t0pp","place":9,"score":3.9,"version":"T0pp"}],"stars":"","top_company":"openai","top_model":"GPT-4","top_model_family":"GPT","top_model_id":"gpt-4","top_model_version":"GPT-4","top_open_source":0,"top_percent":22.6,"topic":"Long-context summarization","total_models":1613,"url":"https://github.com/allenai/scrolls"},{"bench_models":8,"description":"ZeroSCROLLS split based on the SummScreenFD summarization benchmark.","github":"","has_recent":0,"id":"zeroscrolls-summscreenfd","leaderboard":"","lower_is_better":false,"metric":"rgeo","name":"ZeroSCROLLS SummScreenFD","paper":"","placements":[{"company":"google","family":"CoLT5","model":"colt5","place":1,"score":20,"version":"CoLT5"},{"company":"openai","family":"GPT","model":"gpt-4","place":2,"score":17.3,"version":"GPT-4"},{"company":"openai","family":"Davinci","model":"davinci-003","place":3,"score":16.1,"version":"Davinci"},{"company":"openai","family":"GPT","model":"gpt-3.5-turbo-0301","place":4,"score":16.1,"version":"GPT-3.5"},{"company":"anthropic","family":"Claude","model":"claude-v1.3","place":5,"score":16.1,"version":"Claude v1.3"},{"company":"google","family":"FLAN-UL2","model":"flan-ul2","place":6,"score":11.5,"version":"FLAN-UL2"},{"company":"bigscience","family":"T0","model":"t0pp","place":7,"score":9.6,"version":"T0pp"},{"company":"google","family":"FLAN-T5","model":"flan-t5","place":8,"score":7.8,"version":"FLAN-T5"}],"stars":"","top_company":"google","top_model":"CoLT5","top_model_family":"CoLT5","top_model_id":"colt5","top_model_version":"CoLT5","top_open_source":1,"top_percent":20,"topic":"Long-context summarization","total_models":1613,"url":"https://github.com/allenai/scrolls"}]; window.LLM_BOB_MODELS = {"acereason-nemotron-1.1":{"country":"US","id":"acereason-nemotron-1.1","launched":"2025-06-16","modalities":["text"],"model":"AceReason Nemotron","model_family":"AceReason","model_variant":"Nemotron-1.1","model_version":"1.1","name":"AceReason-Nemotron-1.1","open_source":true,"parameter_size":"7B","provider":"nvidia","url":""},"afm-4.5b":{"country":"US","id":"afm-4.5b","launched":"2025-07-29","modalities":["text"],"model":"AFM 4.5B","model_family":"AFM","model_variant":"4.5B","model_version":"AFM","name":"AFM-4.5B","open_source":true,"parameter_size":"4.26B","provider":"arcee","url":""},"alpaca-13b":{"country":"US","id":"alpaca-13b","launched":"","modalities":["text"],"model":"Alpaca 13B","model_family":"Alpaca","model_variant":"Alpaca 13B","model_version":"Alpaca","name":"Alpaca 13B","open_source":true,"parameter_size":"13B","provider":"stanford","url":""},"alpaca-7b":{"country":"US","id":"alpaca-7b","launched":"","modalities":["text"],"model":"Alpaca 7B","model_family":"Alpaca","model_variant":"Alpaca 7B","model_version":"Alpaca","name":"Alpaca 7B","open_source":true,"parameter_size":"7B","provider":"stanford","url":""},"alpaca-farm-ppo-human-7b":{"country":"US","id":"alpaca-farm-ppo-human-7b","launched":"","modalities":["text"],"model":"Alpaca Farm PPO Human 7B","model_family":"Alpaca Farm","model_variant":"PPO Human 7B","model_version":"Alpaca Farm","name":"Alpaca Farm PPO Human 7B","open_source":true,"parameter_size":"7B","provider":"stanford","url":""},"alpaca-farm-ppo-sim-gpt4-7b":{"country":"US","id":"alpaca-farm-ppo-sim-gpt4-7b","launched":"","modalities":["text"],"model":"Alpaca Farm PPO Sim GPT-4 7B","model_family":"Alpaca Farm","model_variant":"PPO Sim GPT-4 7B","model_version":"Alpaca Farm","name":"Alpaca Farm PPO Sim (GPT-4) 7B","open_source":true,"parameter_size":"7B","provider":"stanford","url":""},"amazon-nova-experimental-chat-05-14":{"country":"US","id":"amazon-nova-experimental-chat-05-14","launched":"2025-05-14","modalities":["text"],"model":"Amazon Nova Experimental","model_family":"Amazon Nova","model_variant":"Nova Experimental Chat","model_version":"Amazon Nova","name":"Amazon Nova Experimental Chat (05-14)","open_source":false,"parameter_size":"","provider":"amazon","url":""},"amazon-nova-lite-v1.0":{"country":"US","id":"amazon-nova-lite-v1.0","launched":"2025-12-01","modalities":["text"],"model":"Amazon Nova Lite","model_family":"Amazon Nova","model_variant":"Nova Lite","model_version":"Amazon Nova","name":"Amazon Nova Lite v1.0","open_source":false,"parameter_size":"","provider":"amazon","url":""},"amazon-nova-micro-v1.0":{"country":"US","id":"amazon-nova-micro-v1.0","launched":"2025-12-01","modalities":["text"],"model":"Amazon Nova Micro","model_family":"Amazon Nova","model_variant":"Nova Micro","model_version":"Amazon Nova","name":"Amazon Nova Micro v1.0","open_source":false,"parameter_size":"","provider":"amazon","url":""},"amazon-nova-premier":{"country":"US","id":"amazon-nova-premier","launched":"","modalities":["text"],"model":"Amazon Nova Premier","model_family":"Amazon Nova","model_variant":"Nova Premier","model_version":"Amazon Nova","name":"Amazon Nova Premier","open_source":false,"parameter_size":"","provider":"amazon","url":""},"amazon-nova-pro-v1.0":{"country":"US","id":"amazon-nova-pro-v1.0","launched":"2025-12-01","modalities":["text"],"model":"Amazon Nova Pro","model_family":"Amazon Nova","model_variant":"Nova Pro","model_version":"Amazon Nova","name":"Amazon Nova Pro v1.0","open_source":false,"parameter_size":"","provider":"amazon","url":""},"antgroup-finix-s1-32b":{"country":"CN","id":"antgroup-finix-s1-32b","launched":"","modalities":["text"],"model":"Finix S1","model_family":"Finix","model_variant":"AntGroup Finix_S1_32b","model_version":"Finix S1","name":"AntGroup Finix_S1_32b","open_source":true,"parameter_size":"32B","provider":"antgroup","url":""},"apertus-70b":{"country":"CH","id":"apertus-70b","launched":"2025-09-02","modalities":["text"],"model":"Apertus 70B","model_family":"Apertus","model_variant":"Apertus-70B","model_version":"Apertus","name":"Apertus-70B","open_source":"plus","parameter_size":"70B","provider":"swissai","url":""},"apertus-70b-instruct":{"country":"CH","id":"apertus-70b-instruct","launched":"2025-09-02","modalities":["text"],"model":"Apertus 70B","model_family":"Apertus","model_variant":"Apertus-70B","model_version":"Apertus","name":"Apertus-70B-Instruct","open_source":"plus","parameter_size":"70B","provider":"swissai","url":""},"apriel-1.5-15b-thinker":{"country":"US","id":"apriel-1.5-15b-thinker","launched":"2025-09-30","modalities":["text"],"model":"Apriel 1.5 15B Thinker","model_family":"Apriel","model_variant":"1.5-15B Thinker","model_version":"Apriel 1.5","name":"Apriel-1.5-15B-Thinker","open_source":true,"parameter_size":"15B","provider":"servicenow","url":""},"apriel-1.5-15b-thinker-self-reported":{"country":"US","id":"apriel-1.5-15b-thinker-self-reported","launched":"","modalities":["text"],"model":"Apriel 1.5 15B Thinker","model_family":"Apriel","model_variant":"1.5-15B Thinker (Self-Reported)","model_version":"Apriel 1.5","name":"Apriel-1.5-15B-Thinker (Self-Reported)","open_source":true,"parameter_size":"15B","provider":"servicenow","url":""},"apriel-1.6-15b-thinker":{"country":"US","id":"apriel-1.6-15b-thinker","launched":"","modalities":["text"],"model":"Apriel 1.6 15B Thinker","model_family":"Apriel","model_variant":"1.6-15B Thinker","model_version":"Apriel 1.6","name":"Apriel-1.6-15B-Thinker","open_source":true,"parameter_size":"15B","provider":"servicenow","url":""},"apriel-5b-instruct":{"country":"US","id":"apriel-5b-instruct","launched":"2025-04-11","modalities":["text"],"model":"Apriel 5B","model_family":"Apriel","model_variant":"5B-Instruct","model_version":"Apriel","name":"Apriel-5B-Instruct","open_source":true,"parameter_size":"4.8B","provider":"servicenow","url":""},"apriel-nemotron-15b-thinker":{"country":"US","id":"apriel-nemotron-15b-thinker","launched":"2025-09-30","modalities":["text"],"model":"Apriel Nemotron 15B Thinker","model_family":"Apriel","model_variant":"Nemotron 15B Thinker","model_version":"Apriel Nemotron","name":"Apriel Nemotron 15B Thinker","open_source":true,"parameter_size":"15B","provider":"servicenow","url":""},"aquif-3-0.4b":{"country":"US","id":"aquif-3-0.4b","launched":"2025-07-29","modalities":["text"],"model":"Aquif-3 0.4B","model_family":"Aquif","model_variant":"3-0.4B","model_version":"Aquif 3","name":"Aquif-3 0.4B","open_source":true,"parameter_size":"0.4B","provider":"aquif","url":""},"aquif-3-3.2b":{"country":"US","id":"aquif-3-3.2b","launched":"2025-09-29","modalities":["text"],"model":"Aquif-3 3.2B","model_family":"Aquif","model_variant":"3-3.2B","model_version":"Aquif 3","name":"Aquif-3 3.2B","open_source":true,"parameter_size":"3.2B","provider":"aquif","url":""},"aquif-3-moe-17b-a2.8b":{"active_parameters":"2.8B","country":"US","id":"aquif-3-moe-17b-a2.8b","launched":"2025-08-31","modalities":["text"],"model":"Aquif-3 MoE","model_family":"Aquif","model_variant":"3-MoE-17B-A2.8B","model_version":"Aquif 3","name":"Aquif-3-MoE 17B A2.8B","open_source":true,"parameter_size":"17B","provider":"aquif","url":""},"aquif-3.5-2.7b":{"country":"US","id":"aquif-3.5-2.7b","launched":"2025-08-30","modalities":["text"],"model":"Aquif-3.5 2.7B","model_family":"Aquif","model_variant":"3.5-2.7B","model_version":"Aquif 3.5","name":"Aquif-3.5 2.7B","open_source":true,"parameter_size":"2.7B","provider":"aquif","url":""},"aquif-3.5-a0.6b":{"active_parameters":"0.6B","country":"US","id":"aquif-3.5-a0.6b","launched":"2025-08-30","modalities":["text"],"model":"Aquif-3.5 A0.6B","model_family":"Aquif","model_variant":"3.5-A0.6B","model_version":"Aquif 3.5","name":"Aquif-3.5 A0.6B","open_source":true,"parameter_size":"2.6B","provider":"aquif","url":""},"aquif-3.5-nano-1b":{"country":"US","id":"aquif-3.5-nano-1b","launched":"2025-12-02","modalities":["text"],"model":"Aquif-3.5 Nano 1B","model_family":"Aquif","model_variant":"3.5-Nano-1B","model_version":"Aquif 3.5","name":"Aquif-3.5 Nano 1B","open_source":true,"parameter_size":"1B","provider":"aquif","url":""},"aquif-3.6-1b":{"country":"US","id":"aquif-3.6-1b","launched":"2025-12-05","modalities":["text"],"model":"Aquif-3.6 1B","model_family":"Aquif","model_variant":"3.6-1B","model_version":"Aquif 3.6","name":"Aquif-3.6 1B","open_source":true,"parameter_size":"1B","provider":"aquif","url":""},"architects":{"country":"US","id":"architects","launched":"2024-04-01","modalities":["text"],"model":"ARChitects","model_family":"ARChitects","model_variant":"ARChitects","model_version":"ARChitects","name":"ARChitects","open_source":false,"parameter_size":"","provider":"arcprize","url":""},"artigenz-coder-ds-6.7b":{"country":"US","id":"artigenz-coder-ds-6.7b","launched":"","modalities":["text"],"model":"Artigenz-Coder-DS-6.7B","model_family":"Artigenz-Coder-DS-6.7B","model_variant":"Artigenz-Coder-DS-6.7B","model_version":"Artigenz Coder DS","name":"Artigenz-Coder-DS-6.7B","open_source":true,"parameter_size":"6.7B","provider":"artigenz","url":""},"athene-70b":{"country":"US","id":"athene-70b","launched":"","modalities":["text"],"model":"Athene 70B","model_family":"Athene","model_variant":"Athene 70B","model_version":"Athene V2","name":"Athene 70B","open_source":false,"parameter_size":"70B","provider":"nvidia","url":""},"athene-70b-0725":{"country":"US","id":"athene-70b-0725","launched":"2024-07-25","modalities":["text"],"model":"Athene 70B","model_family":"Athene","model_variant":"Athene 70B","model_version":"Athene","name":"Athene 70B (07-25)","open_source":true,"parameter_size":"70B","provider":"nexusflow","url":""},"athene-v2-agent":{"active_parameters":"72.0B","country":"US","id":"athene-v2-agent","launched":"","modalities":["text"],"model":"Athene V2","model_family":"Athene","model_variant":"Athene V2 Agent","model_version":"Athene V2","name":"Athene-V2-Agent","open_source":false,"parameter_size":"72.0B","provider":"nvidia","url":""},"athene-v2-chat":{"active_parameters":"72.0B","country":"US","id":"athene-v2-chat","launched":"","modalities":["text"],"model":"Athene V2","model_family":"Athene","model_variant":"Athene V2 Chat","model_version":"Athene V2","name":"Athene-V2-Chat","open_source":false,"parameter_size":"72.0B","provider":"nvidia","url":""},"autocoder":{"country":"US","id":"autocoder","launched":"","modalities":["text"],"model":"Autocoder","model_family":"Autocoder","model_variant":"Autocoder","model_version":"Autocoder","name":"Autocoder","open_source":true,"parameter_size":"","provider":"autocoder","url":""},"autocoder-qw-7b":{"country":"US","id":"autocoder-qw-7b","launched":"","modalities":["text"],"model":"Autocoder QW 7B","model_family":"Autocoder","model_variant":"Autocoder QW 7B","model_version":"Autocoder","name":"Autocoder QW 7B","open_source":true,"parameter_size":"7B","provider":"autocoder","url":""},"autocoder-s-6.7b":{"country":"US","id":"autocoder-s-6.7b","launched":"","modalities":["text"],"model":"Autocoder S 6.7B","model_family":"Autocoder","model_variant":"Autocoder S 6.7B","model_version":"Autocoder","name":"Autocoder S 6.7B","open_source":true,"parameter_size":"6.7B","provider":"autocoder","url":""},"ax-k1":{"active_parameters":"33B","country":"KR","id":"ax-k1","launched":"2025-12-31","modalities":["text"],"model":"A.X K1","model_family":"A.X","model_variant":"K1","model_version":"A.X","name":"A.X K1","open_source":true,"parameter_size":"519B","provider":"skt","url":""},"aya-expanse-32b":{"country":"CA","id":"aya-expanse-32b","launched":"","modalities":["text"],"model":"Aya Expanse 32B","model_family":"Aya","model_variant":"Aya Expanse 32B","model_version":"Aya Expanse","name":"Aya Expanse 32B","open_source":true,"parameter_size":"32B","provider":"cohere","url":""},"aya-expanse-8b":{"country":"CA","id":"aya-expanse-8b","launched":"","modalities":["text"],"model":"Aya Expanse 8B","model_family":"Aya","model_variant":"Aya Expanse 8B","model_version":"Aya Expanse","name":"Aya Expanse 8B","open_source":true,"parameter_size":"8B","provider":"cohere","url":""},"bagel":{"country":"US","id":"bagel","launched":"","modalities":["visual"],"model":"Bagel","model_family":"Bagel","model_variant":"Bagel","model_version":"Bagel","name":"BAGEL","open_source":true,"parameter_size":"","provider":"jondurbin","url":""},"bagel-14b":{"country":"US","id":"bagel-14b","launched":"","modalities":["visual"],"model":"Bagel 14B","model_family":"Bagel","model_variant":"Bagel 14B","model_version":"Bagel","name":"Bagel 14B","open_source":true,"parameter_size":"14B","provider":"jondurbin","url":""},"baguettotron-321m":{"country":"FR","id":"baguettotron-321m","launched":"2025-11-11","modalities":["text"],"model":"Baguettotron-321M","model_family":"Baguettotron","model_variant":"Baguettotron-321M","model_version":"Baguettotron","name":"Baguettotron-321M","open_source":true,"parameter_size":"0.321B","provider":"pleias","url":"https://huggingface.co/PleIAs/Baguettotron"},"belle-7b-2m":{"country":"CN","id":"belle-7b-2m","launched":"","modalities":["text"],"model":"BELLE-7B-2M","model_family":"BELLE","model_variant":"BELLE","model_version":"BELLE 7B","name":"BELLE-7B-2M","open_source":true,"parameter_size":"7B","provider":"belle","url":""},"bielik-11b-v2":{"country":"PL","id":"bielik-11b-v2","launched":"","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v2","model_version":"Bielik v2","name":"Bielik-11B-v2","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v2.0-instruct":{"country":"PL","id":"bielik-11b-v2.0-instruct","launched":"","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v2.0 Instruct","model_version":"Bielik v2.0","name":"Bielik-11B-v2.0-Instruct","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v2.1-instruct":{"country":"PL","id":"bielik-11b-v2.1-instruct","launched":"","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v2.1 Instruct","model_version":"Bielik v2.1","name":"Bielik-11B-v2.1-Instruct","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v2.2-instruct":{"country":"PL","id":"bielik-11b-v2.2-instruct","launched":"","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v2.2 Instruct","model_version":"Bielik v2.2","name":"Bielik-11B-v2.2-Instruct","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v2.3-instruct":{"country":"PL","id":"bielik-11b-v2.3-instruct","launched":"","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v2.3 Instruct","model_version":"Bielik v2.3","name":"Bielik-11B-v2.3-Instruct","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v2.5-instruct":{"country":"PL","id":"bielik-11b-v2.5-instruct","launched":"","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v2.5 Instruct","model_version":"Bielik v2.5","name":"Bielik-11B-v2.5-Instruct","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v2.6-instruct":{"country":"PL","id":"bielik-11b-v2.6-instruct","launched":"","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v2.6 Instruct","model_version":"Bielik v2.6","name":"Bielik-11B-v2.6-Instruct","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v3.0":{"country":"PL","id":"bielik-11b-v3.0","launched":"2025-12-01","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v3.0","model_version":"Bielik v3.0","name":"Bielik-11B-v3.0","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-11b-v3.0-instruct":{"country":"PL","id":"bielik-11b-v3.0-instruct","launched":"2025-12-01","modalities":["text"],"model":"Bielik 11B","model_family":"Bielik","model_variant":"11B v3.0 Instruct","model_version":"Bielik v3.0","name":"Bielik-11B-v3.0-Instruct","open_source":true,"parameter_size":"11B","provider":"SpeakLeash","url":""},"bielik-4.5b-v3":{"country":"PL","id":"bielik-4.5b-v3","launched":"","modalities":["text"],"model":"Bielik 4.5B","model_family":"Bielik","model_variant":"4.5B v3","model_version":"Bielik v3","name":"Bielik-4.5B-v3","open_source":true,"parameter_size":"4.5B","provider":"SpeakLeash","url":""},"bielik-4.5b-v3-instruct":{"country":"PL","id":"bielik-4.5b-v3-instruct","launched":"","modalities":["text"],"model":"Bielik 4.5B","model_family":"Bielik","model_variant":"4.5B v3 Instruct","model_version":"Bielik v3","name":"Bielik-4.5B-v3-Instruct","open_source":true,"parameter_size":"4.5B","provider":"SpeakLeash","url":""},"bielik-7b-instruct-v0.1":{"country":"PL","id":"bielik-7b-instruct-v0.1","launched":"","modalities":["text"],"model":"Bielik 7B","model_family":"Bielik","model_variant":"7B v0.1 Instruct","model_version":"Bielik v0.1","name":"Bielik-7B-Instruct-v0.1","open_source":true,"parameter_size":"7B","provider":"SpeakLeash","url":""},"bielik-7b-v0.1":{"country":"PL","id":"bielik-7b-v0.1","launched":"","modalities":["text"],"model":"Bielik 7B","model_family":"Bielik","model_variant":"7B v0.1","model_version":"Bielik v0.1","name":"Bielik-7B-v0.1","open_source":true,"parameter_size":"7B","provider":"SpeakLeash","url":""},"bitnet-2b-4t":{"country":"US","id":"bitnet-2b-4t","launched":"2025-02-18","modalities":["text"],"model":"BitNet b1.58 2B4T","model_family":"BitNet","model_variant":"b1.58-2B4T","model_version":"BitNet b1.58","name":"BitNet b1.58 2B4T","open_source":true,"parameter_size":"2B","provider":"microsoft","url":""},"bitnet-3b":{"country":"US","id":"bitnet-3b","launched":"","modalities":["text"],"model":"BitNet b1.58 3B","model_family":"BitNet","model_variant":"b1.58-3B","model_version":"BitNet b1.58","name":"BitNet b1.58 3B","open_source":true,"parameter_size":"3B","provider":"microsoft","url":""},"blip-3-4b":{"country":"US","id":"blip-3-4b","launched":"2025-05-23","modalities":["visual"],"model":"BLIP-3 4B","model_family":"BLIP","model_variant":"BLIP-3 4B","model_version":"BLIP-3","name":"BLIP-3 4B","open_source":true,"parameter_size":"4B","provider":"salesforce","url":""},"blip-3o-4b":{"country":"US","id":"blip-3o-4b","launched":"2025-05-23","modalities":["visual"],"model":"BLIP-3O 4B","model_family":"BLIP","model_variant":"BLIP-3O 4B","model_version":"BLIP-3O","name":"BLIP-3O 4B","open_source":true,"parameter_size":"4B","provider":"salesforce","url":""},"bloom-176b":{"country":"US","id":"bloom-176b","launched":"2022-07-06","modalities":["text"],"model":"BLOOM","model_family":"BLOOM","model_variant":"176B","model_version":"176B","name":"BLOOM-176B","open_source":"plus","parameter_size":"176B","provider":"bigscience","url":"https://huggingface.co/bigscience/bloom"},"blt-7b":{"country":"US","id":"blt-7b","launched":"2025-04-17","modalities":["text"],"model":"BLT 7B","model_family":"BLT","model_variant":"BLT 7B","model_version":"BLT 7B","name":"BLT 7B","open_source":true,"parameter_size":"7B","provider":"meta","url":""},"bolmo-7b":{"country":"US","id":"bolmo-7b","launched":"2025-12-15","modalities":["text"],"model":"Bolmo 7B","model_family":"Bolmo","model_variant":"Bolmo 7B","model_version":"Bolmo 7B","name":"Bolmo 7B","open_source":true,"parameter_size":"7B","provider":"allenai","url":""},"bunny-4b":{"country":"CN","id":"bunny-4b","launched":"2024-05-08","modalities":["visual"],"model":"Bunny 4B","model_family":"Bunny","model_variant":"Bunny 4B","model_version":"Bunny","name":"Bunny 4B","open_source":true,"parameter_size":"4B","provider":"baai","url":""},"c4ai-aya-expanse-32b":{"country":"CA","id":"c4ai-aya-expanse-32b","launched":"","modalities":["text"],"model":"Aya Expanse 32B","model_family":"Aya","model_variant":"Aya Expanse 32B","model_version":"Aya Expanse","name":"C4AI Aya Expanse 32B","open_source":true,"parameter_size":"32B","provider":"cohere","url":""},"c4ai-aya-expanse-8b":{"country":"CA","id":"c4ai-aya-expanse-8b","launched":"","modalities":["text"],"model":"Aya Expanse 8B","model_family":"Aya","model_variant":"Aya Expanse 8B","model_version":"Aya Expanse","name":"C4AI Aya Expanse 8B","open_source":true,"parameter_size":"8B","provider":"cohere","url":""},"c4ai-aya-vision-32b":{"country":"CA","id":"c4ai-aya-vision-32b","launched":"","modalities":["visual"],"model":"Aya Vision 32B","model_family":"Aya","model_variant":"Aya Vision 32B","model_version":"Aya Vision","name":"C4AI Aya Vision 32B","open_source":"plus","parameter_size":"32B","provider":"cohere","url":""},"c4ai-command-r-08-2024":{"country":"CA","id":"c4ai-command-r-08-2024","launched":"","modalities":["text"],"model":"C4ai Command R 08 2024","model_family":"C4ai","model_variant":"C4ai Command R 08 2024","model_version":"C4ai","name":"C4ai Command R 08 2024","open_source":false,"parameter_size":"","provider":"cohere","url":""},"c4ai-command-r-08.2024":{"country":"CA","id":"c4ai-command-r-08.2024","launched":"","modalities":["text"],"model":"C4ai Command R 08.2024","model_family":"C4ai","model_variant":"C4ai Command R 08.2024","model_version":"C4ai","name":"C4ai Command R 08.2024","open_source":false,"parameter_size":"","provider":"cohere","url":""},"c4ai-command-r-plus-08.2024":{"country":"CA","id":"c4ai-command-r-plus-08.2024","launched":"","modalities":["text"],"model":"C4ai Command R Plus 08.2024","model_family":"C4ai","model_variant":"C4ai Command R Plus 08.2024","model_version":"C4ai","name":"C4ai Command R Plus 08.2024","open_source":false,"parameter_size":"","provider":"cohere","url":""},"cambrian-1-34b":{"country":"US","id":"cambrian-1-34b","launched":"","modalities":["visual"],"model":"Cambrian-1 34B","model_family":"Cambrian","model_variant":"Cambrian-1-34B","model_version":"Cambrian 1","name":"Cambrian-1-34B","open_source":"plus","parameter_size":"34B","provider":"nyu","url":""},"cambrian-1-8b":{"country":"US","id":"cambrian-1-8b","launched":"","modalities":["visual"],"model":"Cambrian-1 8B","model_family":"Cambrian","model_variant":"Cambrian-1-8B","model_version":"Cambrian 1","name":"Cambrian-1-8B","open_source":"plus","parameter_size":"8B","provider":"nyu","url":""},"cambrian-34b":{"country":"US","id":"cambrian-34b","launched":"","modalities":["visual"],"model":"Cambrian 34B","model_family":"Cambrian","model_variant":"Cambrian 34B","model_version":"Cambrian","name":"Cambrian 34B","open_source":true,"parameter_size":"34B","provider":"nyu","url":""},"chandra-ocr-0.1.0":{"country":"US","id":"chandra-ocr-0.1.0","launched":"","modalities":["visual"],"model":"Chandra OCR 0.1.0","model_family":"Chandra OCR","model_variant":"0.1.0","model_version":"Chandra OCR","name":"Chandra OCR 0.1.0","open_source":true,"parameter_size":"","provider":"chandra","url":""},"chatglm-6b":{"country":"CN","id":"chatglm-6b","launched":"","modalities":["text"],"model":"ChatGLM 6B","model_family":"ChatGLM","model_variant":"ChatGLM 6B","model_version":"ChatGLM","name":"ChatGLM 6B","open_source":true,"parameter_size":"6B","provider":"zai","url":""},"chatglm2-6b":{"country":"CN","id":"chatglm2-6b","launched":"","modalities":["text"],"model":"ChatGLM2-6B","model_family":"ChatGLM","model_variant":"ChatGLM2-6B","model_version":"ChatGLM2","name":"ChatGLM2-6B","open_source":true,"parameter_size":"6B","provider":"zai","url":""},"chatglm3-6b":{"country":"CN","id":"chatglm3-6b","launched":"","modalities":["text"],"model":"ChatGLM3 6B","model_family":"ChatGLM","model_variant":"ChatGLM3 6B","model_version":"ChatGLM3","name":"ChatGLM3 6B","open_source":true,"parameter_size":"6B","provider":"zai","url":""},"chatgpt":{"country":"US","id":"chatgpt","launched":"","modalities":["text"],"model":"ChatGPT","model_family":"GPT","model_variant":"ChatGPT","model_version":"ChatGPT","name":"ChatGPT","open_source":false,"parameter_size":"1760Be","provider":"openai","url":""},"chatgpt-4o":{"country":"US","id":"chatgpt-4o","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"GPT-4o","model_version":"GPT-4o","name":"ChatGPT-4o","open_source":false,"parameter_size":"","provider":"openai","url":""},"chatgpt-4o-latest-20250129":{"country":"US","id":"chatgpt-4o-latest-20250129","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"ChatGPT 4o latest 20250129","model_version":"GPT-4o","name":"ChatGPT-4o-latest-20250129","open_source":false,"parameter_size":"","provider":"openai","url":""},"chatgpt-4o-latest-20250215":{"country":"US","id":"chatgpt-4o-latest-20250215","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"ChatGPT 4o latest 20250215","model_version":"GPT-4o","name":"ChatGPT-4o-latest (2025-02-15)","open_source":false,"parameter_size":"","provider":"openai","url":""},"chatgpt-4o-latest-20250326":{"country":"US","id":"chatgpt-4o-latest-20250326","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"ChatGPT-4o Latest","model_version":"GPT-4o","name":"ChatGPT-4o Latest (2025-03-26)","open_source":false,"parameter_size":"","provider":"openai","url":""},"chatgpt-4o-latest-20250329":{"country":"US","id":"chatgpt-4o-latest-20250329","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"ChatGPT 4o latest 20250329","model_version":"GPT-4o","name":"ChatGPT-4o-latest (2025-03-29)","open_source":false,"parameter_size":"","provider":"openai","url":""},"chatgpt-agent":{"country":"US","id":"chatgpt-agent","launched":"","modalities":["text"],"model":"ChatGPT Agent","model_family":"GPT","model_variant":"ChatGPT Agent","model_version":"ChatGPT","name":"ChatGPT Agent","open_source":false,"parameter_size":"","provider":"openai","url":""},"chinchilla-70b":{"country":"US","id":"chinchilla-70b","launched":"2022-03-29","modalities":["text"],"model":"Chinchilla 70B","model_family":"Chinchilla","model_variant":"70B","model_version":"Chinchilla","name":"Chinchilla 70B","open_source":false,"parameter_size":"70B","provider":"google","url":""},"claude":{"country":"US","id":"claude","launched":"2023-03-14","modalities":["text"],"model":"Claude","model_family":"Claude","model_variant":"Claude 1","model_version":"Claude 1","name":"Claude","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-2":{"country":"US","id":"claude-2","launched":"2023-07-11","modalities":["text"],"model":"claude-2 (Mar 2024)","model_family":"claude-2","model_variant":"claude-2 (Mar 2024)","model_version":"claude-2 (Mar 2024)","name":"claude-2 (Mar 2024)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-2.1":{"country":"US","id":"claude-2.1","launched":"","modalities":["text"],"model":"Claude 2.1","model_family":"Claude","model_variant":"Claude 2.1","model_version":"Claude 2","name":"Claude 2.1","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-5-haiku-20240620":{"country":"US","id":"claude-3-5-haiku-20240620","launched":"2024-10-22","modalities":["text"],"model":"Claude 3.5 Haiku","model_family":"Claude","model_variant":"Claude 3.5 Haiku","model_version":"Claude 3.5","name":"Claude 3.5 Haiku (2024-06-20)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-5-haiku-20241022":{"country":"US","id":"claude-3-5-haiku-20241022","launched":"2024-10-22","modalities":["text"],"model":"Claude 3.5 Haiku","model_family":"Claude","model_variant":"Claude 3.5 Haiku","model_version":"Claude 3.5","name":"Claude 3.5 Haiku (2024-10-22)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-5-sonnet-20240620":{"country":"US","id":"claude-3-5-sonnet-20240620","launched":"2024-06-20","modalities":["text"],"model":"Claude 3.5 Sonnet","model_family":"Claude","model_variant":"Claude 3.5 Sonnet","model_version":"Claude 3.5","name":"Claude 3.5 Sonnet (2024-06-20)","open_source":false,"parameter_size":"130Be","provider":"anthropic","url":""},"claude-3-5-sonnet-20241022":{"country":"US","id":"claude-3-5-sonnet-20241022","launched":"2024-10-22","modalities":["text"],"model":"Claude 3.5 Sonnet","model_family":"Claude","model_variant":"Claude 3.5 Sonnet","model_version":"Claude 3.5","name":"Claude 3.5 Sonnet (2024-10-22)","open_source":false,"parameter_size":"130Be","provider":"anthropic","url":""},"claude-3-7-sonnet-20250219":{"country":"US","id":"claude-3-7-sonnet-20250219","launched":"2025-02-19","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude 3.7 Sonnet","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet (2025-02-19)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-7-sonnet-20250219-thinking-16k":{"country":"US","id":"claude-3-7-sonnet-20250219-thinking-16k","launched":"2025-02-19","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude 3.7 Sonnet Thinking 16k","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet (2025-02-19) Thinking 16k","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-7-sonnet-20250219-thinking-32k":{"country":"US","id":"claude-3-7-sonnet-20250219-thinking-32k","launched":"2025-02-19","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude 3.7 Sonnet Thinking","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet (2025-02-19) Thinking 32k","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-haiku":{"country":"US","id":"claude-3-haiku","launched":"2024-03-04","modalities":["text"],"model":"Claude 3 Haiku","model_family":"Claude","model_variant":"Claude 3","model_version":"Claude 3","name":"Claude 3 Haiku","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-haiku-20240307":{"country":"US","id":"claude-3-haiku-20240307","launched":"2024-03-04","modalities":["text"],"model":"Claude 3 Haiku","model_family":"Claude","model_variant":"3 Haiku 20240307","model_version":"Claude 3","name":"Claude 3 Haiku 20240307","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-haiku-mar-2024":{"country":"US","id":"claude-3-haiku-mar-2024","launched":"","modalities":["text"],"model":"claude-3-haiku (Mar 2024)","model_family":"claude-3-haiku","model_variant":"claude-3-haiku (Mar 2024)","model_version":"claude-3-haiku (Mar 2024)","name":"claude-3-haiku (Mar 2024)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-opus":{"country":"US","id":"claude-3-opus","launched":"2024-03-04","modalities":["text","visual"],"model":"Claude 3 Opus","model_family":"Claude","model_variant":"Claude 3 Opus","model_version":"Claude 3","name":"Claude 3 Opus","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-opus-20240229":{"country":"US","id":"claude-3-opus-20240229","launched":"2024-03-04","modalities":["text"],"model":"Claude 3 Opus","model_family":"Claude","model_variant":"Claude 3 Opus 20240229","model_version":"Claude 3","name":"Claude-3-Opus-20240229","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3-sonnet":{"country":"US","id":"claude-3-sonnet","launched":"2024-03-04","modalities":["text"],"model":"Claude Sonnet 3","model_family":"Claude","model_variant":"Claude 3","model_version":"Claude 3","name":"Claude Sonnet 3","open_source":false,"parameter_size":"70Be","provider":"anthropic","url":""},"claude-3-sonnet-20240229":{"country":"US","id":"claude-3-sonnet-20240229","launched":"2024-02-29","modalities":["text"],"model":"Claude 3 Sonnet","model_family":"Claude","model_variant":"3 Sonnet 20240229","model_version":"Claude 3","name":"Claude 3 Sonnet 20240229","open_source":false,"parameter_size":"70Be","provider":"anthropic","url":""},"claude-3-sonnet-mar-2024":{"country":"US","id":"claude-3-sonnet-mar-2024","launched":"2024-03-15","modalities":["text"],"model":"claude-3-sonnet (Mar 2024)","model_family":"claude-3-sonnet","model_variant":"claude-3-sonnet (Mar 2024)","model_version":"claude-3-sonnet (Mar 2024)","name":"claude-3-sonnet (Mar 2024)","open_source":false,"parameter_size":"70Be","provider":"anthropic","url":""},"claude-3.5-haiku":{"country":"US","id":"claude-3.5-haiku","launched":"2024-10-22","modalities":["text"],"model":"Claude 3.5 Haiku","model_family":"Claude","model_variant":"Claude3.5","model_version":"Claude 3.5","name":"Claude 3.5 Haiku","open_source":false,"parameter_size":"40Be","provider":"anthropic","url":""},"claude-3.5-haiku-20241022":{"country":"US","id":"claude-3.5-haiku-20241022","launched":"2024-10-22","modalities":["text"],"model":"Claude 3.5 Haiku","model_family":"Claude","model_variant":"Claude 3.5 Haiku","model_version":"Claude 3.5","name":"Claude 3.5 Haiku (2024-10-22)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3.5-sonnet":{"country":"US","id":"claude-3.5-sonnet","launched":"2024-06-20","modalities":["text"],"model":"Claude 3.5 Sonnet","model_family":"Claude","model_variant":"Claude3.5","model_version":"Claude 3.5","name":"Claude 3.5 Sonnet","open_source":false,"parameter_size":"130Be","provider":"anthropic","url":""},"claude-3.5-sonnet-20240620":{"country":"US","id":"claude-3.5-sonnet-20240620","launched":"2024-06-20","modalities":["text"],"model":"Claude 3.5 Sonnet","model_family":"Claude","model_variant":"3.5 Sonnet 20240620","model_version":"Claude 3.5","name":"Claude 3.5 Sonnet 20240620","open_source":false,"parameter_size":"130Be","provider":"anthropic","url":""},"claude-3.5-sonnet-20241022":{"country":"US","id":"claude-3.5-sonnet-20241022","launched":"2024-10-22","modalities":["text"],"model":"Claude 3.5 Sonnet","model_family":"Claude","model_variant":"Claude3.5 Sonnet 20241022","model_version":"Claude 3.5","name":"Claude 3.5 Sonnet (20241022)","open_source":false,"parameter_size":"130Be","provider":"anthropic","url":""},"claude-3.7-sonnet":{"country":"US","id":"claude-3.7-sonnet","launched":"2025-02-24","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude3.7","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet","open_source":false,"parameter_size":"200Be","provider":"anthropic","url":""},"claude-3.7-sonnet-20250219":{"country":"US","id":"claude-3.7-sonnet-20250219","launched":"2025-02-19","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude 3.7 Sonnet","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet (2025-02-19)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3.7-sonnet-thinking":{"country":"US","id":"claude-3.7-sonnet-thinking","launched":"2025-02-24","modalities":["text"],"model":"Claude 3.7 Sonnet Thinking","model_family":"Claude","model_variant":"Claude 3.7 Sonnet Thinking","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet Thinking","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3.7-sonnet-thinking-16k":{"country":"US","id":"claude-3.7-sonnet-thinking-16k","launched":"2025-02-24","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude 3.7 Sonnet Thinking 16k","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet Thinking (16k)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3.7-sonnet-thinking-1k":{"country":"US","id":"claude-3.7-sonnet-thinking-1k","launched":"2025-02-24","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude 3.7 Sonnet Thinking 1k","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet Thinking (1k)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-3.7-sonnet-thinking-8k":{"country":"US","id":"claude-3.7-sonnet-thinking-8k","launched":"2025-02-24","modalities":["text"],"model":"Claude 3.7 Sonnet","model_family":"Claude","model_variant":"Claude 3.7 Sonnet Thinking 8k","model_version":"Claude 3.7","name":"Claude 3.7 Sonnet Thinking (8k)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-4":{"country":"US","id":"claude-4","launched":"2025-05-22","modalities":["text"],"model":"Claude 4","model_family":"Claude","model_variant":"4","model_version":"Claude 4","name":"Claude 4","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-4-opus":{"country":"US","id":"claude-4-opus","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4","model_family":"Claude","model_variant":"Claude 4 Opus","model_version":"Claude 4","name":"Claude 4 Opus","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-4-opus-thinking":{"country":"US","id":"claude-4-opus-thinking","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4 Thinking","model_family":"Claude","model_variant":"Claude 4 Opus Thinking","model_version":"Claude 4","name":"Claude 4 Opus Thinking","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-4-sonet":{"country":"US","id":"claude-4-sonet","launched":"2025-05-22","modalities":["text"],"model":"Claude 4 Sonnet","model_family":"Claude","model_variant":"4 Sonnet","model_version":"Claude 4","name":"Claude 4 Sonnet","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-4-sonnet":{"country":"US","id":"claude-4-sonnet","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4","model_version":"Claude 4","name":"Claude 4 Sonnet","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-4.1-opus-thinking":{"country":"US","id":"claude-4.1-opus-thinking","launched":"","modalities":["text"],"model":"Claude Opus 4.1 Thinking","model_family":"Claude","model_variant":"Claude 4.1 Opus Thinking","model_version":"Claude 4.1","name":"Claude 4.1 Opus Thinking","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-4.5-haiku":{"country":"US","id":"claude-4.5-haiku","launched":"2025-10-15","modalities":["text"],"model":"Claude Haiku 4.5","model_family":"Claude","model_variant":"Claude Haiku 4.5","model_version":"Claude 4.5","name":"Claude Haiku 4.5","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-4.5-sonnet":{"country":"US","id":"claude-4.5-sonnet","launched":"2025-09-29","modalities":["text"],"model":"Claude Sonnet 4.5","model_family":"Claude","model_variant":"Claude Sonnet 4.5","model_version":"Claude 4.5","name":"Claude Sonnet 4.5","open_source":false,"parameter_size":"150Be","provider":"anthropic","url":""},"claude-4.5-sonnet-thinking":{"country":"US","id":"claude-4.5-sonnet-thinking","launched":"2025-09-29","modalities":["text"],"model":"Claude Sonnet 4.5","model_family":"Claude","model_variant":"Claude Sonnet 4.5 Thinking","model_version":"Claude 4.5","name":"Claude Sonnet 4.5 (Thinking)","open_source":false,"parameter_size":"150Be","provider":"anthropic","url":""},"claude-4.5-sonnet-thinking-16k":{"country":"US","id":"claude-4.5-sonnet-thinking-16k","launched":"2025-09-29","modalities":["text"],"model":"Claude Sonnet 4.5","model_family":"Claude","model_variant":"Claude Sonnet 4.5 Thinking 16k","model_version":"Claude 4.5","name":"Claude Sonnet 4.5 (Thinking 16k)","open_source":false,"parameter_size":"150Be","provider":"anthropic","url":""},"claude-4.5-sonnet-thinking-1k":{"country":"US","id":"claude-4.5-sonnet-thinking-1k","launched":"2025-09-29","modalities":["text"],"model":"Claude Sonnet 4.5","model_family":"Claude","model_variant":"Claude Sonnet 4.5 Thinking 1k","model_version":"Claude 4.5","name":"Claude Sonnet 4.5 (Thinking 1k)","open_source":false,"parameter_size":"150Be","provider":"anthropic","url":""},"claude-4.5-sonnet-thinking-32k":{"country":"US","id":"claude-4.5-sonnet-thinking-32k","launched":"2025-09-29","modalities":["text"],"model":"Claude Sonnet 4.5","model_family":"Claude","model_variant":"Claude Sonnet 4.5 Thinking 32k","model_version":"Claude 4.5","name":"Claude Sonnet 4.5 (Thinking 32k)","open_source":false,"parameter_size":"150Be","provider":"anthropic","url":""},"claude-4.5-sonnet-thinking-8k":{"country":"US","id":"claude-4.5-sonnet-thinking-8k","launched":"2025-09-29","modalities":["text"],"model":"Claude Sonnet 4.5","model_family":"Claude","model_variant":"Claude Sonnet 4.5 Thinking 8k","model_version":"Claude 4.5","name":"Claude Sonnet 4.5 (Thinking 8k)","open_source":false,"parameter_size":"150Be","provider":"anthropic","url":""},"claude-instant-1-mar-2024":{"country":"US","id":"claude-instant-1-mar-2024","launched":"","modalities":["text"],"model":"claude-instant-1 (Mar 2024)","model_family":"claude-instant-1","model_variant":"claude-instant-1 (Mar 2024)","model_version":"claude-instant-1 (Mar 2024)","name":"claude-instant-1 (Mar 2024)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-3":{"country":"US","id":"claude-opus-3","launched":"2024-03-04","modalities":["text"],"model":"Claude Opus 3","model_family":"Claude","model_variant":"Claude 3","model_version":"Claude 3","name":"Claude Opus 3","open_source":false,"parameter_size":"2000Be","provider":"anthropic","url":""},"claude-opus-4":{"country":"US","id":"claude-opus-4","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4","model_family":"Claude","model_variant":"Claude","model_version":"Claude 4","name":"Claude Opus 4","open_source":false,"parameter_size":"1000Be","provider":"anthropic","url":""},"claude-opus-4-1-20250805":{"country":"US","id":"claude-opus-4-1-20250805","launched":"2025-08-05","modalities":["text"],"model":"Claude Opus 4.1","model_family":"Claude","model_variant":"Claude Opus 4.1","model_version":"Claude 4.1","name":"Claude Opus 4.1 (2025-08-05)","open_source":false,"parameter_size":"400Be","provider":"anthropic","url":""},"claude-opus-4-1-20250805-thinking":{"country":"US","id":"claude-opus-4-1-20250805-thinking","launched":"2025-08-05","modalities":["text"],"model":"Claude Opus 4.1","model_family":"Claude","model_variant":"Claude Opus 4.1 Thinking","model_version":"Claude 4.1","name":"Claude Opus 4.1 (2025-08-05) Thinking","open_source":false,"parameter_size":"400Be","provider":"anthropic","url":""},"claude-opus-4-1-20250805-thinking-16k":{"country":"US","id":"claude-opus-4-1-20250805-thinking-16k","launched":"2025-08-05","modalities":["text"],"model":"Claude Opus 4.1","model_family":"Claude","model_variant":"Claude Opus 4.1 Thinking","model_version":"Claude 4.1","name":"Claude Opus 4.1 (2025-08-05) Thinking 16k","open_source":false,"parameter_size":"400Be","provider":"anthropic","url":""},"claude-opus-4-20250514":{"country":"US","id":"claude-opus-4-20250514","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4","model_family":"Claude","model_variant":"Claude Opus 4","model_version":"Claude 4","name":"Claude Opus 4 (2025-05-14)","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4-20250514-thinking-16k":{"country":"US","id":"claude-opus-4-20250514-thinking-16k","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4","model_family":"Claude","model_variant":"Claude Opus 4 Thinking","model_version":"Claude 4","name":"Claude Opus 4 (2025-05-14) Thinking 16k","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4-20250514-thinking-1k":{"country":"US","id":"claude-opus-4-20250514-thinking-1k","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4","model_family":"Claude","model_variant":"Claude Opus 4 Thinking 1k","model_version":"Claude 4","name":"Claude Opus 4 (2025-05-14) Thinking 1k","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4-20250514-thinking-8k":{"country":"US","id":"claude-opus-4-20250514-thinking-8k","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4","model_family":"Claude","model_variant":"Claude Opus 4 Thinking 8k","model_version":"Claude 4","name":"Claude Opus 4 (2025-05-14) Thinking 8k","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4-cot":{"country":"US","id":"claude-opus-4-cot","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4 CoT","model_family":"Claude","model_variant":"Claude Opus 4 CoT","model_version":"Claude 4","name":"Claude Opus 4 CoT","open_source":false,"parameter_size":"1000Be","provider":"anthropic","url":""},"claude-opus-4-thinking":{"country":"US","id":"claude-opus-4-thinking","launched":"2025-05-22","modalities":["text"],"model":"Claude Opus 4","model_family":"Claude","model_variant":"Opus 4 Thinking","model_version":"Claude 4","name":"Claude Opus 4 (Thinking)","open_source":false,"parameter_size":"1000Be","provider":"anthropic","url":""},"claude-opus-4.1":{"country":"US","id":"claude-opus-4.1","launched":"2025-08-05","modalities":["text"],"model":"Claude Opus 4.1","model_family":"Claude","model_variant":"Claude","model_version":"Claude 4.1","name":"Claude Opus 4.1","open_source":false,"parameter_size":"400Be","provider":"anthropic","url":""},"claude-opus-4.1-20250805":{"country":"US","id":"claude-opus-4.1-20250805","launched":"2025-08-05","modalities":["text"],"model":"Claude Opus 4.1","model_family":"Claude","model_variant":"Claude Opus 4.1","model_version":"Claude 4.1","name":"Claude Opus 4.1 (2025-08-05)","open_source":false,"parameter_size":"400Be","provider":"anthropic","url":""},"claude-opus-4.5":{"country":"US","id":"claude-opus-4.5","launched":"2025-11-24","modalities":["text"],"model":"Claude Opus 4.5","model_family":"Claude","model_variant":"Claude","model_version":"Claude 4.5","name":"Claude Opus 4.5","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4.5-max":{"country":"US","id":"claude-opus-4.5-max","launched":"2025-11-24","modalities":["text"],"model":"Claude Opus 4.5","model_family":"Claude","model_variant":"Claude Opus 4.5 Max","model_version":"Claude 4.5","name":"Claude Opus 4.5 Max","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4.5-medium":{"country":"US","id":"claude-opus-4.5-medium","launched":"2025-11-24","modalities":["text"],"model":"Claude Opus 4.5","model_family":"Claude","model_variant":"Claude Opus 4.5 Medium","model_version":"Claude 4.5","name":"Claude Opus 4.5 Medium","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4.5-thinking":{"country":"US","id":"claude-opus-4.5-thinking","launched":"2025-11-24","modalities":["text"],"model":"Claude Opus 4.5","model_family":"Claude","model_variant":"Claude Opus 4.5 Thinking","model_version":"Claude 4.5","name":"Claude Opus 4.5 Thinking","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-opus-4.6":{"country":"US","id":"claude-opus-4.6","launched":"2026-02-11","modalities":["text"],"model":"Claude Opus 4.6","model_family":"Claude","model_variant":"Claude Opus 4.6","model_version":"Claude 4.6","name":"Claude Opus 4.6","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-sonnet-3.5":{"country":"US","id":"claude-sonnet-3.5","launched":"","modalities":["text"],"model":"Claude 3.5 Sonnet","model_family":"Claude","model_variant":"Claude3.5","model_version":"Claude 3.5","name":"Claude Sonnet 3.5","open_source":false,"parameter_size":"130Be","provider":"anthropic","url":""},"claude-sonnet-4":{"country":"US","id":"claude-sonnet-4","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude","model_version":"Claude 4","name":"Claude Sonnet 4","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-20250514":{"country":"US","id":"claude-sonnet-4-20250514","launched":"2025-05-14","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4","model_version":"Claude 4","name":"Claude Sonnet 4 (2025-05-14)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-20250514-thinking-16k":{"country":"US","id":"claude-sonnet-4-20250514-thinking-16k","launched":"2025-05-14","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4 Thinking 16k","model_version":"Claude 4","name":"Claude Sonnet 4 (2025-05-14) Thinking 16k","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-20250514-thinking-1k":{"country":"US","id":"claude-sonnet-4-20250514-thinking-1k","launched":"2025-05-14","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4 Thinking 1k","model_version":"Claude 4","name":"Claude Sonnet 4 (2025-05-14) Thinking 1k","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-20250514-thinking-32k":{"country":"US","id":"claude-sonnet-4-20250514-thinking-32k","launched":"2025-05-14","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4 Thinking","model_version":"Claude 4","name":"Claude Sonnet 4 (2025-05-14) Thinking 32k","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-20250514-thinking-8k":{"country":"US","id":"claude-sonnet-4-20250514-thinking-8k","launched":"2025-05-14","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4 Thinking 8k","model_version":"Claude 4","name":"Claude Sonnet 4 (2025-05-14) Thinking 8k","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-moatless-tools":{"country":"US","id":"claude-sonnet-4-moatless-tools","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Sonnet 4 Moatless Tools","model_version":"Claude 4","name":"Claude Sonnet 4 (Moatless Tools)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-r2e-gym":{"country":"US","id":"claude-sonnet-4-r2e-gym","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Sonnet 4 R2E-Gym","model_version":"Claude 4","name":"Claude Sonnet 4 (R2E-Gym)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-swe-agent":{"country":"US","id":"claude-sonnet-4-swe-agent","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Sonnet 4 SWE-Agent","model_version":"Claude 4","name":"Claude Sonnet 4 (SWE-Agent)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-thinking":{"country":"US","id":"claude-sonnet-4-thinking","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Sonnet 4 Thinking","model_version":"Claude 4","name":"Claude Sonnet 4 (t)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-thinking-16k":{"country":"US","id":"claude-sonnet-4-thinking-16k","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4 Thinking 16k","model_version":"Claude 4","name":"Claude Sonnet 4 (Thinking 16k)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-thinking-1k":{"country":"US","id":"claude-sonnet-4-thinking-1k","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4 Thinking 1k","model_version":"Claude 4","name":"Claude Sonnet 4 (Thinking 1k)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4-thinking-8k":{"country":"US","id":"claude-sonnet-4-thinking-8k","launched":"2025-05-22","modalities":["text"],"model":"Claude Sonnet 4","model_family":"Claude","model_variant":"Claude Sonnet 4 Thinking 8k","model_version":"Claude 4","name":"Claude Sonnet 4 (Thinking 8k)","open_source":false,"parameter_size":"100Be","provider":"anthropic","url":""},"claude-sonnet-4.5":{"country":"US","id":"claude-sonnet-4.5","launched":"2025-11-24","modalities":["text"],"model":"Claude Sonnet 4.5","model_family":"Claude","model_variant":"Claude Sonnet 4.5","model_version":"Claude 4.5","name":"Claude Sonnet 4.5","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-sonnet-4.6":{"country":"US","id":"claude-sonnet-4.6","launched":"2026-02-17","modalities":["text"],"model":"Claude Sonnet 4.6","model_family":"Claude","model_variant":"Claude Sonnet 4.6","model_version":"Claude 4.6","name":"Claude Sonnet 4.6","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"claude-v1.3":{"country":"US","id":"claude-v1.3","launched":"2023-04-18","modalities":["text"],"model":"Claude v1.3","model_family":"Claude","model_variant":"Claude v1.3","model_version":"Claude v1.3","name":"Claude v1.3","open_source":false,"parameter_size":"","provider":"anthropic","url":""},"coda-1.7b-base":{"country":"US","id":"coda-1.7b-base","launched":"2025-09-25","modalities":["text"],"model":"CoDA 1.7B","model_family":"CoDA","model_variant":"CoDA 1.7B Base","model_version":"CoDA","name":"CoDA-1.7B-Base","open_source":"plus","parameter_size":"1.7B","provider":"salesforce","url":""},"coda-1.7b-instruct":{"country":"US","id":"coda-1.7b-instruct","launched":"2025-09-25","modalities":["text"],"model":"CoDA 1.7B","model_family":"CoDA","model_variant":"CoDA 1.7B Instruct","model_version":"CoDA","name":"CoDA-1.7B-Instruct","open_source":"plus","parameter_size":"1.7B","provider":"salesforce","url":""},"code-13b":{"country":"US","id":"code-13b","launched":"","modalities":["text"],"model":"Code-13B","model_family":"Code-13B","model_variant":"Code-13B","model_version":"Code-13B","name":"Code-13B","open_source":true,"parameter_size":"13B","provider":"code","url":""},"code-290k-6.7b-instruct":{"country":"US","id":"code-290k-6.7b-instruct","launched":"","modalities":["text"],"model":"Code-290k-6.7B","model_family":"Code","model_variant":"Code-290k-6.7B-Instruct","model_version":"Code 290k","name":"Code-290k-6.7B-Instruct","open_source":true,"parameter_size":"6.7B","provider":"code","url":""},"code-33b":{"country":"US","id":"code-33b","launched":"","modalities":["text"],"model":"Code-33B","model_family":"Code-33B","model_variant":"Code-33B","model_version":"Code-33B","name":"Code-33B","open_source":true,"parameter_size":"33B","provider":"code","url":""},"code-millenials-34b":{"country":"US","id":"code-millenials-34b","launched":"","modalities":["text"],"model":"code-millenials-34B","model_family":"code-millenials-34B","model_variant":"code-millenials-34B","model_version":"code-millenials-34B","name":"code-millenials-34B","open_source":true,"parameter_size":"34B","provider":"code","url":""},"codegeex4-all-9b":{"country":"CN","id":"codegeex4-all-9b","launched":"","modalities":["text"],"model":"Codegeex4 ALL 9B","model_family":"Codegeex4","model_variant":"Codegeex4 ALL 9B","model_version":"Codegeex4","name":"Codegeex4 ALL 9B","open_source":true,"parameter_size":"9B","provider":"zhongguancun","url":""},"codegemma-1.1-7b":{"country":"US","id":"codegemma-1.1-7b","launched":"2024-02-22","modalities":["text"],"model":"CodeGemma 1.1 7B","model_family":"CodeGemma","model_variant":"CodeGemma 1.1 7B","model_version":"CodeGemma","name":"CodeGemini 1.1 7B","open_source":true,"parameter_size":"7B","provider":"google","url":""},"codegemma-2b":{"country":"US","id":"codegemma-2b","launched":"","modalities":["text"],"model":"codegemma-2b","model_family":"Codegemma","model_variant":"codegemma-2b","model_version":"codegemma-2b","name":"codegemma-2b","open_source":true,"parameter_size":"2B","provider":"google","url":""},"codegemma-7b":{"country":"US","id":"codegemma-7b","launched":"","modalities":["text"],"model":"codegemma-7b","model_family":"Codegemma","model_variant":"codegemma-7b","model_version":"codegemma-7b","name":"codegemma-7b","open_source":true,"parameter_size":"7B","provider":"google","url":""},"codegemma-7b-instruct":{"country":"US","id":"codegemma-7b-instruct","launched":"","modalities":["text"],"model":"Codegemma 7B","model_family":"codegemma-2b","model_variant":"Codegemma 7B Instruct","model_version":"Codegemma","name":"Codegemma 7B Instruct","open_source":true,"parameter_size":"7B","provider":"codegemma","url":""},"codegemma-7b-it":{"country":"US","id":"codegemma-7b-it","launched":"","modalities":["text"],"model":"Codegemma 7b","model_family":"Codegemma","model_variant":"codegemma-7b-it","model_version":"Codegemma","name":"codegemma-7b-it","open_source":true,"parameter_size":"7B","provider":"google","url":""},"codegen-16b":{"country":"US","id":"codegen-16b","launched":"","modalities":["text"],"model":"CodeGen-16B","model_family":"CodeGen","model_variant":"CodeGen-16B","model_version":"CodeGen","name":"CodeGen-16B","open_source":true,"parameter_size":"16B","provider":"salesforce","url":""},"codegen-2b":{"country":"US","id":"codegen-2b","launched":"","modalities":["text"],"model":"CodeGen-2B","model_family":"CodeGen","model_variant":"CodeGen-2B","model_version":"CodeGen","name":"CodeGen-2B","open_source":true,"parameter_size":"2B","provider":"salesforce","url":""},"codegen-6b":{"country":"US","id":"codegen-6b","launched":"","modalities":["text"],"model":"CodeGen-6B","model_family":"CodeGen","model_variant":"CodeGen-6B","model_version":"CodeGen","name":"CodeGen-6B","open_source":true,"parameter_size":"6B","provider":"salesforce","url":""},"codegen2-16b":{"country":"US","id":"codegen2-16b","launched":"","modalities":["text"],"model":"CodeGen2-16B","model_family":"CodeGen","model_variant":"CodeGen2-16B","model_version":"CodeGen2","name":"CodeGen2-16B","open_source":true,"parameter_size":"16B","provider":"salesforce","url":""},"codegen2-1b":{"country":"US","id":"codegen2-1b","launched":"","modalities":["text"],"model":"CodeGen2-1B","model_family":"CodeGen","model_variant":"CodeGen2-1B","model_version":"CodeGen2","name":"CodeGen2-1B","open_source":true,"parameter_size":"1B","provider":"salesforce","url":""},"codegen2-3b":{"country":"US","id":"codegen2-3b","launched":"","modalities":["text"],"model":"CodeGen2-3B","model_family":"CodeGen","model_variant":"CodeGen2-3B","model_version":"CodeGen2","name":"CodeGen2-3B","open_source":true,"parameter_size":"3B","provider":"salesforce","url":""},"codegen2-7b":{"country":"US","id":"codegen2-7b","launched":"","modalities":["text"],"model":"CodeGen2-7B","model_family":"CodeGen","model_variant":"CodeGen2-7B","model_version":"CodeGen2","name":"CodeGen2-7B","open_source":true,"parameter_size":"7B","provider":"salesforce","url":""},"codellama-13b":{"country":"US","id":"codellama-13b","launched":"","modalities":["text"],"model":"CodeLlama-13B","model_family":"CodeLlama-13B","model_variant":"CodeLlama-13B","model_version":"CodeLlama-13B","name":"CodeLlama-13B","open_source":true,"parameter_size":"13B","provider":"codellama","url":""},"codellama-13b-cot":{"country":"US","id":"codellama-13b-cot","launched":"","modalities":["text"],"model":"CodeLlama 13B CoT","model_family":"CodeLlama","model_variant":"CodeLlama 13B CoT","model_version":"CodeLlama","name":"CodeLlama 13B CoT","open_source":true,"parameter_size":"13B","provider":"codellama","url":""},"codellama-13b-instruct":{"country":"US","id":"codellama-13b-instruct","launched":"","modalities":["text"],"model":"Codellama 13B","model_family":"CodeLlama","model_variant":"Codellama 13B Instruct","model_version":"CodeLlama","name":"Codellama 13B Instruct","open_source":true,"parameter_size":"13B","provider":"meta","url":""},"codellama-34b":{"country":"US","id":"codellama-34b","launched":"","modalities":["text"],"model":"CodeLlama-34B","model_family":"CodeLlama-34B","model_variant":"CodeLlama-34B","model_version":"CodeLlama-34B","name":"CodeLlama-34B","open_source":true,"parameter_size":"34B","provider":"codellama","url":""},"codellama-34b-cot":{"country":"US","id":"codellama-34b-cot","launched":"","modalities":["text"],"model":"CodeLlama 34B CoT","model_family":"CodeLlama","model_variant":"CodeLlama 34B CoT","model_version":"CodeLlama","name":"CodeLlama 34B CoT","open_source":true,"parameter_size":"34B","provider":"codellama","url":""},"codellama-34b-instruct":{"country":"US","id":"codellama-34b-instruct","launched":"","modalities":["text"],"model":"Codellama 34B","model_family":"CodeLlama","model_variant":"Codellama 34B Instruct","model_version":"CodeLlama","name":"Codellama 34B Instruct","open_source":true,"parameter_size":"34B","provider":"meta","url":""},"codellama-70b":{"country":"US","id":"codellama-70b","launched":"","modalities":["text"],"model":"CodeLlama 70B","model_family":"CodeLlama","model_variant":"70B","model_version":"CodeLlama","name":"CodeLlama 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"codellama-70b-instruct":{"country":"US","id":"codellama-70b-instruct","launched":"","modalities":["text"],"model":"CodeLlama 70B","model_family":"CodeLlama","model_variant":"CodeLlama-70B-Instruct","model_version":"CodeLlama","name":"CodeLlama-70B-Instruct","open_source":true,"parameter_size":"70B","provider":"codellama","url":""},"codellama-7b":{"country":"US","id":"codellama-7b","launched":"2024-12-06","modalities":["text"],"model":"CodeLlama-7B","model_family":"CodeLlama-7B","model_variant":"CodeLlama-7B","model_version":"CodeLlama-7B","name":"CodeLlama-7B","open_source":true,"parameter_size":"7B","provider":"codellama","url":""},"codellama-7b-cot":{"country":"US","id":"codellama-7b-cot","launched":"","modalities":["text"],"model":"CodeLlama 7B CoT","model_family":"CodeLlama","model_variant":"CodeLlama 7B CoT","model_version":"CodeLlama","name":"CodeLlama 7B CoT","open_source":true,"parameter_size":"7B","provider":"codellama","url":""},"codellama-7b-instruct":{"country":"US","id":"codellama-7b-instruct","launched":"","modalities":["text"],"model":"Codellama 7B","model_family":"CodeLlama","model_variant":"Codellama 7B Instruct","model_version":"CodeLlama","name":"Codellama 7B Instruct","open_source":true,"parameter_size":"7B","provider":"meta","url":""},"codellama-python-13b":{"country":"US","id":"codellama-python-13b","launched":"","modalities":["text"],"model":"CodeLlama Python 13B","model_family":"CodeLlama","model_variant":"Python 13B","model_version":"CodeLlama","name":"CodeLlama Python 13B","open_source":true,"parameter_size":"13B","provider":"codellama","url":""},"codellama-python-34b":{"country":"US","id":"codellama-python-34b","launched":"","modalities":["text"],"model":"CodeLlama Python 34B","model_family":"CodeLlama","model_variant":"Python 34B","model_version":"CodeLlama","name":"CodeLlama Python 34B","open_source":true,"parameter_size":"34B","provider":"codellama","url":""},"codellama-python-7b":{"country":"US","id":"codellama-python-7b","launched":"","modalities":["text"],"model":"CodeLlama Python 7B","model_family":"CodeLlama","model_variant":"Python 7B","model_version":"CodeLlama","name":"CodeLlama Python 7B","open_source":true,"parameter_size":"7B","provider":"codellama","url":""},"codeqwen1.5-7b":{"country":"CN","id":"codeqwen1.5-7b","launched":"","modalities":["text"],"model":"CodeQwen1.5 7B","model_family":"Qwen","model_variant":"CodeQwen1.5-7B","model_version":"CodeQwen1.5","name":"CodeQwen1.5-7B","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"codeqwen1.5-7b-chat":{"country":"CN","id":"codeqwen1.5-7b-chat","launched":"","modalities":["text"],"model":"CodeQwen1.5 7B","model_family":"Qwen","model_variant":"CodeQwen1.5-7B-Chat","model_version":"CodeQwen1.5","name":"CodeQwen1.5-7B-Chat","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"codestral-12b-instruct":{"country":"FR","id":"codestral-12b-instruct","launched":"","modalities":["text"],"model":"Codestral 12B Instruct","model_family":"Codestral","model_variant":"Codestral 12B Instruct","model_version":"Codestral","name":"Codestral 12B Instruct","open_source":true,"parameter_size":"12B","provider":"mistral","url":""},"codestral-22b":{"country":"FR","id":"codestral-22b","launched":"2024-05-22","modalities":["text"],"model":"Codestral 22B","model_family":"Codestral","model_variant":"Mistral-22B","model_version":"Codestral","name":"Codestral 22B","open_source":true,"parameter_size":"22B","provider":"mistral","url":""},"codestral-22b-v0.1":{"country":"FR","id":"codestral-22b-v0.1","launched":"","modalities":["text"],"model":"Codestral 22B v0.1","model_family":"Codestral","model_variant":"Codestral v0.1","model_version":"Codestral v0.1","name":"Codestral-22B-v0.1","open_source":true,"parameter_size":"22B","provider":"mistral","url":""},"codestral-2501":{"country":"FR","id":"codestral-2501","launched":"2025-01-15","modalities":["text"],"model":"Codestral 22B","model_family":"Codestral","model_variant":"Codestral 22B","model_version":"Codestral","name":"Codestral 2501","open_source":true,"parameter_size":"22B","provider":"mistral","url":""},"codestral-2508":{"country":"FR","id":"codestral-2508","launched":"2025-08-01","modalities":["text"],"model":"Codestral 22B","model_family":"Codestral","model_variant":"Codestral 22B","model_version":"Codestral","name":"Codestral 2508","open_source":true,"parameter_size":"22B","provider":"mistral","url":""},"codestral-mamba":{"country":"FR","id":"codestral-mamba","launched":"2024-07-16","modalities":["text"],"model":"Codestral Mamba","model_family":"Codestral","model_variant":"Codestral Mamba","model_version":"Codestral","name":"Codestral Mamba","open_source":true,"parameter_size":"7B","provider":"mistral","published_date":"2024-07-16","url":""},"codet5-16b":{"country":"US","id":"codet5-16b","launched":"","modalities":["text"],"model":"CodeT5+-16B","model_family":"CodeT5+-16B","model_variant":"CodeT5+-16B","model_version":"CodeT5+-16B","name":"CodeT5+-16B","open_source":true,"parameter_size":"16B","provider":"salesforce","url":""},"codet5-2b":{"country":"US","id":"codet5-2b","launched":"","modalities":["text"],"model":"CodeT5+-2B","model_family":"CodeT5+-2B","model_variant":"CodeT5+-2B","model_version":"CodeT5+-2B","name":"CodeT5+-2B","open_source":true,"parameter_size":"2B","provider":"salesforce","url":""},"codet5-6b":{"country":"US","id":"codet5-6b","launched":"","modalities":["text"],"model":"CodeT5+-6B","model_family":"CodeT5+-6B","model_variant":"CodeT5+-6B","model_version":"CodeT5+-6B","name":"CodeT5+-6B","open_source":true,"parameter_size":"6B","provider":"salesforce","url":""},"codet5plus-16b":{"country":"US","id":"codet5plus-16b","launched":"","modalities":["text"],"model":"Codet5plus 16B","model_family":"Codet5plus","model_variant":"Codet5plus 16B","model_version":"Codet5plus","name":"Codet5plus 16B","open_source":true,"parameter_size":"16B","provider":"salesforce","url":""},"codet5plus-6b":{"country":"US","id":"codet5plus-6b","launched":"","modalities":["text"],"model":"Codet5plus 6B","model_family":"Codet5plus","model_variant":"Codet5plus 6B","model_version":"Codet5plus","name":"Codet5plus 6B","open_source":true,"parameter_size":"6B","provider":"salesforce","url":""},"codetulu-2-34b":{"country":"US","id":"codetulu-2-34b","launched":"","modalities":["text"],"model":"CodeTulu-2 34B","model_family":"CodeTulu","model_variant":"CodeTulu-2 34B","model_version":"CodeTulu","name":"CodeTulu-2 34B","open_source":true,"parameter_size":"34B","provider":"codetulu","url":""},"codex-mini-latest":{"country":"US","id":"codex-mini-latest","launched":"2025-05-16","modalities":["text"],"model":"Codex Mini","model_family":"Codex","model_variant":"Codex Mini Latest","model_version":"Codex","name":"Codex Mini (Latest)","open_source":false,"parameter_size":"","provider":"openai","url":""},"cogito-671b-v2.1":{"country":"US","id":"cogito-671b-v2.1","launched":"2025-11-18","modalities":["text"],"model":"Cogito 671B","model_family":"Cogito","model_variant":"671B-v2.1","model_version":"Cogito 671B v2.1","name":"Cogito-671B-v2.1","open_source":true,"parameter_size":"671B","provider":"deepcogito","url":""},"cogvlm2-llama3-chat-19b":{"country":"CN","id":"cogvlm2-llama3-chat-19b","launched":"","modalities":["visual"],"model":"CogVLM2","model_family":"CogVLM2","model_variant":"Llama3 Chat 19B","model_version":"CogVLM2","name":"CogVLM2 Llama3 Chat 19B","open_source":true,"parameter_size":"19B","provider":"zai","url":""},"colt5":{"country":"US","id":"colt5","launched":"","modalities":["text"],"model":"CoLT5","model_family":"CoLT5","model_variant":"CoLT5","model_version":"CoLT5","name":"CoLT5","open_source":true,"parameter_size":"","provider":"google","url":""},"command":{"country":"CA","id":"command","launched":"2024-08-01","modalities":["text"],"model":"Cohere Command","model_family":"Command","model_variant":"Command","model_version":"Command","name":"Cohere Command","open_source":false,"parameter_size":"","provider":"cohere","url":""},"command-a":{"country":"CA","id":"command-a","launched":"","modalities":["text"],"model":"Command A","model_family":"Command","model_variant":"Command A","model_version":"Command","name":"Cohere Command A","open_source":false,"parameter_size":"111B","provider":"cohere","url":""},"command-a-03-2025":{"country":"CA","id":"command-a-03-2025","launched":"2025-03-01","modalities":["text"],"model":"Command-A","model_family":"Command","model_variant":"Command-A","model_version":"Command","name":"Command-A (03-2025)","open_source":true,"parameter_size":"","provider":"cohere","url":""},"command-a-03-2025-quality":{"country":"CA","id":"command-a-03-2025-quality","launched":"2025-03-01","modalities":["text"],"model":"Command A Quality","model_family":"Command","model_variant":"Command A 2025-03 Quality","model_version":"Command","name":"Command A (03-2025 Quality)","open_source":false,"parameter_size":"","provider":"cohere","url":""},"command-r":{"country":"CA","id":"command-r","launched":"","modalities":["text"],"model":"Command R","model_family":"Command","model_variant":"Command R","model_version":"Command","name":"Command R","open_source":false,"parameter_size":"","provider":"cohere","url":""},"command-r-08-2024":{"country":"CA","id":"command-r-08-2024","launched":"2024-08-01","modalities":["text"],"model":"Command R","model_family":"Command","model_variant":"Command R","model_version":"Command","name":"Command R (08-2024)","open_source":true,"parameter_size":"35B","provider":"cohere","url":""},"command-r-plus":{"country":"CA","id":"command-r-plus","launched":"","modalities":["text"],"model":"Command R+","model_family":"Command","model_variant":"Command R+","model_version":"Command","name":"Command R+","open_source":false,"parameter_size":"","provider":"cohere","url":""},"command-r-plus-08-2024":{"country":"CA","id":"command-r-plus-08-2024","launched":"2024-08-01","modalities":["text"],"model":"Command R Plus","model_family":"Command","model_variant":"Command R Plus","model_version":"Command","name":"Command R Plus (08-2024)","open_source":true,"parameter_size":"104B","provider":"cohere","url":""},"command-r7b":{"country":"CA","id":"command-r7b","launched":"","modalities":["text"],"model":"Command R7B","model_family":"Command","model_variant":"Command-7B","model_version":"Command","name":"Command R7B","open_source":true,"parameter_size":"7B","provider":"cohere","url":""},"cwm-32b":{"country":"US","id":"cwm-32b","launched":"","modalities":["text"],"model":"CWM 32B","model_family":"CWM","model_variant":"32B","model_version":"CWM","name":"CWM-32B","open_source":true,"parameter_size":"32B","provider":"cwm","url":""},"dall-e-3":{"country":"US","id":"dall-e-3","launched":"","modalities":["visual"],"model":"DALL-E 3","model_family":"DALL-E","model_variant":"3","model_version":"DALL-E 3","name":"DALL-E 3","open_source":false,"parameter_size":"","provider":"openai","url":""},"databricks-dbrx-instruct":{"country":"US","id":"databricks-dbrx-instruct","launched":"","modalities":["text"],"model":"databricks/dbrx-instruct","model_family":"databricks/dbrx-instruct","model_variant":"databricks/dbrx-instruct","model_version":"databricks/dbrx-instruct","name":"databricks/dbrx-instruct","open_source":true,"parameter_size":"","provider":"databricks","url":""},"davinci-001":{"country":"US","id":"davinci-001","launched":"","modalities":["text"],"model":"Davinci001","model_family":"Davinci","model_variant":"Davinci 001","model_version":"Davinci","name":"Davinci001","open_source":false,"parameter_size":"","provider":"openai","url":""},"davinci-003":{"country":"US","id":"davinci-003","launched":"2022-11-28","modalities":["text"],"model":"Davinci003","model_family":"Davinci","model_variant":"Davinci 003","model_version":"Davinci","name":"Davinci003","open_source":false,"parameter_size":"","provider":"openai","url":""},"dbrx-instruct-preview":{"country":"US","id":"dbrx-instruct-preview","launched":"","modalities":["text"],"model":"DBRX Instruct","model_family":"DBRX","model_variant":"DBRX Instruct","model_version":"DBRX","name":"DBRX Instruct Preview","open_source":true,"parameter_size":"132B","provider":"databricks","url":""},"deepcoder-14b-preview":{"country":"US","id":"deepcoder-14b-preview","launched":"","modalities":["text"],"model":"Deepcoder 14B Preview","model_family":"Deepcoder","model_variant":"Deepcoder 14B Preview","model_version":"Deepcoder","name":"Deepcoder 14B Preview","open_source":true,"parameter_size":"14B","provider":"deepcoder","url":""},"deepseek-base-1.3b":{"country":"CN","id":"deepseek-base-1.3b","launched":"","modalities":["text"],"model":"DeepSeek 1.3B","model_family":"DeepSeek","model_variant":"1.3B","model_version":"DeepSeek","name":"DeepSeek Base 1.3B","open_source":true,"parameter_size":"1.3B","provider":"deepseek","url":""},"deepseek-base-33b":{"country":"CN","id":"deepseek-base-33b","launched":"","modalities":["text"],"model":"DeepSeek 33B","model_family":"DeepSeek","model_variant":"33B","model_version":"DeepSeek","name":"DeepSeek Base 33B","open_source":true,"parameter_size":"33B","provider":"deepseek","url":""},"deepseek-base-6.7b":{"country":"CN","id":"deepseek-base-6.7b","launched":"","modalities":["text"],"model":"DeepSeek 6.7B","model_family":"DeepSeek","model_variant":"6.7B","model_version":"DeepSeek","name":"DeepSeek Base 6.7B","open_source":true,"parameter_size":"6.7B","provider":"deepseek","url":""},"deepseek-chat-v2.5":{"country":"CN","id":"deepseek-chat-v2.5","launched":"","modalities":["text"],"model":"DeepSeek V2.5","model_family":"DeepSeek","model_variant":"Chat V2.5","model_version":"DeepSeek V2.5","name":"DeepSeek Chat V2.5","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-chat-v3":{"country":"CN","id":"deepseek-chat-v3","launched":"","modalities":["text"],"model":"DeepSeek V3","model_family":"DeepSeek","model_variant":"Chat V3","model_version":"DeepSeek V3","name":"DeepSeek Chat V3","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-chat-v3-prev":{"country":"CN","id":"deepseek-chat-v3-prev","launched":"","modalities":["text"],"model":"DeepSeek V3","model_family":"DeepSeek","model_variant":"Chat V3 Prev","model_version":"DeepSeek V3","name":"DeepSeek Chat V3 (Prev)","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-coder-1.3b-base":{"country":"CN","id":"deepseek-coder-1.3b-base","launched":"","modalities":["text"],"model":"DeepSeek Coder 1.3B","model_family":"DeepSeek","model_variant":"DeepSeek-Coder-1.3B-base","model_version":"DeepSeek Coder","name":"DeepSeek-Coder-1.3B-base","open_source":true,"parameter_size":"1.3B","provider":"deepseek","url":""},"deepseek-coder-1.3b-instruct":{"country":"CN","id":"deepseek-coder-1.3b-instruct","launched":"","modalities":["text"],"model":"DeepSeek Coder 1.3B","model_family":"DeepSeek","model_variant":"DeepSeek-Coder-1.3B-instruct","model_version":"DeepSeek Coder","name":"DeepSeek-Coder-1.3B-instruct","open_source":true,"parameter_size":"1.3B","provider":"deepseek","url":""},"deepseek-coder-33b":{"country":"CN","id":"deepseek-coder-33b","launched":"2024-08-16","modalities":["text"],"model":"DeepSeek Coder 33B","model_family":"DeepSeek Coder","model_variant":"33B","model_version":"DeepSeek Coder","name":"DeepSeek Coder 33B","open_source":true,"parameter_size":"33B","provider":"deepseek","url":""},"deepseek-coder-33b-base":{"country":"CN","id":"deepseek-coder-33b-base","launched":"2024-08-16","modalities":["text"],"model":"DeepSeek Coder 33B","model_family":"DeepSeek","model_variant":"DeepSeek Coder 33B","model_version":"DeepSeek Coder","name":"DeepSeek-Coder-33B-base","open_source":true,"parameter_size":"33B","provider":"deepseek","url":""},"deepseek-coder-33b-instruct":{"country":"CN","id":"deepseek-coder-33b-instruct","launched":"2024-08-16","modalities":["text"],"model":"DeepSeek Coder 33B","model_family":"DeepSeek","model_variant":"DeepSeek-Coder-33B-instruct","model_version":"DeepSeek Coder","name":"DeepSeek-Coder-33B-instruct","open_source":true,"parameter_size":"33B","provider":"deepseek","url":""},"deepseek-coder-6.7b-base":{"country":"CN","id":"deepseek-coder-6.7b-base","launched":"","modalities":["text"],"model":"DeepSeek Coder 6.7B","model_family":"DeepSeek","model_variant":"DeepSeek-Coder-6.7B-base","model_version":"DeepSeek Coder","name":"DeepSeek-Coder-6.7B-base","open_source":true,"parameter_size":"6.7B","provider":"deepseek","url":""},"deepseek-coder-6.7b-instruct":{"country":"CN","id":"deepseek-coder-6.7b-instruct","launched":"","modalities":["text"],"model":"DeepSeek Coder 6.7B","model_family":"DeepSeek","model_variant":"DeepSeek Coder 6.7B Instruct","model_version":"DeepSeek Coder","name":"DeepSeek-Coder-6.7B-instruct","open_source":true,"parameter_size":"6.7B","provider":"deepseek","url":""},"deepseek-coder-7b-instruct-v1.5":{"country":"CN","id":"deepseek-coder-7b-instruct-v1.5","launched":"","modalities":["text"],"model":"DeepSeek Coder 7B v1.5","model_family":"DeepSeek","model_variant":"DeepSeek-Coder-7B-instruct-v1.5","model_version":"DeepSeek Coder","name":"DeepSeek-Coder-7B-instruct-v1.5","open_source":true,"parameter_size":"7B","provider":"deepseek","url":""},"deepseek-coder-v2":{"country":"CN","id":"deepseek-coder-v2","launched":"2025-06-17","modalities":["text"],"model":"DeepSeek Coder V2","model_family":"DeepSeek","model_variant":"DeepSeek Coder V2","model_version":"DeepSeek Coder","name":"DeepSeek Coder V2","open_source":true,"parameter_size":"236B","provider":"deepseek","url":""},"deepseek-coder-v2-instruct":{"country":"CN","id":"deepseek-coder-v2-instruct","launched":"","modalities":["text"],"model":"DeepSeek-Coder V2","model_family":"DeepSeek","model_variant":"DeepSeek-Coder-V2","model_version":"DeepSeek Coder V2","name":"DeepSeek-Coder V2 Instruct","open_source":true,"parameter_size":"236B","provider":"deepseek","url":""},"deepseek-coder-v2-lite-base":{"country":"CN","id":"deepseek-coder-v2-lite-base","launched":"","modalities":["text"],"model":"DeepSeek-Coder V2 Lite","model_family":"DeepSeek","model_variant":"DeepSeek-Coder V2 Lite","model_version":"DeepSeek Coder V2","name":"DeepSeek-Coder V2 Lite Base","open_source":true,"parameter_size":"2.4B","provider":"deepseek","url":""},"deepseek-coder-v2-lite-instruct":{"country":"CN","id":"deepseek-coder-v2-lite-instruct","launched":"","modalities":["text"],"model":"DeepSeek-Coder V2 Lite","model_family":"DeepSeek","model_variant":"DeepSeek-Coder V2 Lite","model_version":"DeepSeek Coder V2","name":"DeepSeek-Coder V2 Lite Instruct","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-instruct-1.3b":{"country":"CN","id":"deepseek-instruct-1.3b","launched":"","modalities":["text"],"model":"DeepSeek 1.3B","model_family":"DeepSeek","model_variant":"1.3B","model_version":"DeepSeek","name":"DeepSeek Instruct 1.3B","open_source":true,"parameter_size":"1.3B","provider":"deepseek","url":""},"deepseek-instruct-33b":{"country":"CN","id":"deepseek-instruct-33b","launched":"","modalities":["text"],"model":"Deepseek 33B","model_family":"DeepSeek","model_variant":"Deepseek Instruct 33B","model_version":"DeepSeek","name":"Deepseek Instruct 33B","open_source":true,"parameter_size":"33B","provider":"deepseek","url":""},"deepseek-instruct-6.7b":{"country":"CN","id":"deepseek-instruct-6.7b","launched":"","modalities":["text"],"model":"DeepSeek 6.7B","model_family":"DeepSeek","model_variant":"6.7B","model_version":"DeepSeek","name":"DeepSeek Instruct 6.7B","open_source":true,"parameter_size":"6.7B","provider":"deepseek","url":""},"deepseek-llm-67b-chat":{"country":"CN","id":"deepseek-llm-67b-chat","launched":"","modalities":["text"],"model":"DeepSeek LLM 67B","model_family":"DeepSeek","model_variant":"DeepSeek LLM 67B","model_version":"DeepSeek","name":"DeepSeek LLM 67B Chat","open_source":true,"parameter_size":"67B","provider":"deepseek","url":""},"deepseek-math-7b":{"country":"CN","id":"deepseek-math-7b","launched":"","modalities":["text"],"model":"DeepSeek Math 7B","model_family":"DeepSeek Math","model_variant":"DeepSeek Math 7B","model_version":"DeepSeek Math","name":"DeepSeek Math 7B","open_source":true,"parameter_size":"7B","provider":"deepseek","url":""},"deepseek-ocr":{"country":"CN","id":"deepseek-ocr","launched":"2025-10-20","modalities":["visual"],"model":"DeepSeek-OCR","model_family":"DeepSeek OCR","model_variant":"DeepSeek-OCR","model_version":"DeepSeek OCR","name":"DeepSeek-OCR","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-ocr-2":{"country":"CN","id":"deepseek-ocr-2","launched":"2026-01-27","modalities":["visual"],"model":"DeepSeek-OCR-2","model_family":"DeepSeek OCR","model_variant":"DeepSeek-OCR-2","model_version":"DeepSeek OCR 2","name":"DeepSeek-OCR-2","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-ocr-base":{"country":"CN","id":"deepseek-ocr-base","launched":"2025-10-20","modalities":["visual"],"model":"DeepSeek-OCR Base","model_family":"DeepSeek OCR","model_variant":"Base","model_version":"DeepSeek OCR","name":"DeepSeek-OCR Base","open_source":true,"parameter_size":"3B","provider":"deepseek","url":""},"deepseek-ocr-large":{"country":"CN","id":"deepseek-ocr-large","launched":"2025-10-20","modalities":["visual"],"model":"DeepSeek-OCR Large","model_family":"DeepSeek OCR","model_variant":"Large","model_version":"DeepSeek OCR","name":"DeepSeek-OCR Large","open_source":true,"parameter_size":"3B","provider":"deepseek","url":""},"deepseek-ocr-small":{"country":"CN","id":"deepseek-ocr-small","launched":"2025-10-20","modalities":["visual"],"model":"DeepSeek-OCR Small","model_family":"DeepSeek OCR","model_variant":"Small","model_version":"DeepSeek OCR","name":"DeepSeek-OCR Small","open_source":true,"parameter_size":"3B","provider":"deepseek","url":""},"deepseek-ocr-tiny":{"country":"CN","id":"deepseek-ocr-tiny","launched":"2025-10-20","modalities":["visual"],"model":"DeepSeek-OCR Tiny","model_family":"DeepSeek OCR","model_variant":"Tiny","model_version":"DeepSeek OCR","name":"DeepSeek-OCR Tiny","open_source":true,"parameter_size":"3B","provider":"deepseek","url":""},"deepseek-r1":{"country":"CN","id":"deepseek-r1","launched":"2025-01-20","modalities":["text"],"model":"DeepSeek R1","model_family":"DeepSeek","model_variant":"DeepSeekR1","model_version":"DeepSeek R1","name":"DeepSeek R1","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-r1-0120":{"country":"CN","id":"deepseek-r1-0120","launched":"2025-01-20","modalities":["text"],"model":"DeepSeek R1","model_family":"DeepSeek","model_variant":"R1-0120","model_version":"0120","name":"DeepSeek R1-0120","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-r1-0528":{"country":"CN","id":"deepseek-r1-0528","launched":"2025-05-28","modalities":["text"],"model":"DeepSeek R1","model_family":"DeepSeek","model_variant":"R1-0528","model_version":"DeepSeek R1","name":"DeepSeek-R1-0528","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-r1-0528-qwen3-8b":{"country":"CN","id":"deepseek-r1-0528-qwen3-8b","launched":"2025-05-28","modalities":["text"],"model":"DeepSeek R1-0528 Qwen3 8B","model_family":"DeepSeek","model_variant":"R1-0528 Qwen3-8B","model_version":"DeepSeek R1","name":"DeepSeek-R1-0528-Qwen3-8B","open_source":true,"parameter_size":"8B","provider":"deepseek","url":""},"deepseek-r1-2025-05-28":{"country":"CN","id":"deepseek-r1-2025-05-28","launched":"","modalities":["text"],"model":"DeepSeek R1","model_family":"DeepSeek","model_variant":"DeepSeek R1 2025-05-28","model_version":"DeepSeek R1","name":"DeepSeek R1 (2025-05-28)","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-r1-70b":{"country":"CN","id":"deepseek-r1-70b","launched":"","modalities":["text"],"model":"DeepSeek R1 70B","model_family":"DeepSeek","model_variant":"R1-70B","model_version":"DeepSeek R1","name":"DeepSeek-R1-70B","open_source":true,"parameter_size":"70B","provider":"deepseek","url":""},"deepseek-r1-distill":{"country":"CN","id":"deepseek-r1-distill","launched":"2025-01-20","modalities":["text"],"model":"DeepSeek R1 Distill","model_family":"DeepSeek","model_variant":"R1 Distill","model_version":"DeepSeek R1","name":"DeepSeek-R1 Distill","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-r1-distill-32b":{"country":"CN","id":"deepseek-r1-distill-32b","launched":"","modalities":["text"],"model":"DeepSeek-R1 Distill","model_family":"DeepSeek-R1","model_variant":"Distill 32B","model_version":"32B","name":"DeepSeek-R1 Distill 32B","open_source":true,"parameter_size":"32B","provider":"deepseek","url":""},"deepseek-r1-distill-llama-70b":{"country":"CN","id":"deepseek-r1-distill-llama-70b","launched":"2025-01-25","modalities":["text"],"model":"DeepSeek R1 Distill Llama 70B","model_family":"DeepSeek","model_variant":"DeepSeek R1 Distill Llama 70B","model_version":"DeepSeek R1","name":"DeepSeek R1 Distill Llama 70B","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-r1-distill-llama-8b":{"country":"CN","id":"deepseek-r1-distill-llama-8b","launched":"2025-01-20","modalities":["text"],"model":"DeepSeek R1 Distill Llama 8B","model_family":"DeepSeek","model_variant":"DeepSeek R1 Distill Llama 8B","model_version":"DeepSeek R1","name":"DeepSeek R1 Distill Llama 8B","open_source":true,"parameter_size":"8B","provider":"deepseek","url":""},"deepseek-r1-distill-qwen-1.5b":{"country":"CN","id":"deepseek-r1-distill-qwen-1.5b","launched":"2025-05-29","modalities":["text"],"model":"DeepSeek R1 Distill Qwen1.5B","model_family":"DeepSeek","model_variant":"R1-Distill-Qwen-1.5B","model_version":"DeepSeek R1","name":"DeepSeek R1 Distill Qwen1.5B","open_source":true,"parameter_size":"1.5B","provider":"deepseek","url":""},"deepseek-r1-distill-qwen-14b":{"country":"CN","id":"deepseek-r1-distill-qwen-14b","launched":"2025-05-29","modalities":["text"],"model":"Deepseek R1 Distill Qwen 14B","model_family":"DeepSeek","model_variant":"Deepseek R1 Distill Qwen 14B","model_version":"DeepSeek R1","name":"Deepseek R1 Distill Qwen 14B","open_source":true,"parameter_size":"14B","provider":"deepseek","url":""},"deepseek-r1-distill-qwen-32b":{"country":"CN","id":"deepseek-r1-distill-qwen-32b","launched":"2025-05-29","modalities":["text"],"model":"DeepSeek R1 Distill Qwen 32B","model_family":"DeepSeek","model_variant":"DeepSeek R1 Distill Qwen 32B","model_version":"DeepSeek R1","name":"DeepSeek R1 Distill Qwen 32B","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-r1-distill-qwen-7b":{"country":"CN","id":"deepseek-r1-distill-qwen-7b","launched":"2025-05-29","modalities":["text"],"model":"DeepSeek R1 Distill Qwen 7B","model_family":"DeepSeek","model_variant":"DeepSeek R1 Distill Qwen 7B","model_version":"DeepSeek R1","name":"DeepSeek R1 Distill Qwen 7B","open_source":true,"parameter_size":"7B","provider":"deepseek","url":""},"deepseek-r1-plus-claude-3-5-sonnet-20241022":{"country":"","id":"deepseek-r1-plus-claude-3-5-sonnet-20241022","launched":"2024-10-22","modalities":["text"],"model":"DeepSeek R1 + Claude 3.5 Sonnet","model_family":"Composite","model_variant":"DeepSeek R1 + Claude 3.5 Sonnet","model_version":"Composite","name":"DeepSeek R1 + Claude 3.5 Sonnet (2024-10-22)","open_source":false,"parameter_size":"","provider":"ensemble","url":""},"deepseek-r1-preview":{"country":"CN","id":"deepseek-r1-preview","launched":"2024-11-20","modalities":["text"],"model":"DeepSeek R1","model_family":"DeepSeek","model_variant":"R1 Preview","model_version":"DeepSeek R1","name":"DeepSeek R1 Preview","open_source":false,"parameter_size":"","provider":"deepseek","url":""},"deepseek-tng-r1t-chimera":{"country":"DE","id":"deepseek-tng-r1t-chimera","launched":"2025-07-03","modalities":["text"],"model":"DeepSeek-TNG R1T Chimera","model_family":"DeepSeek-TNG","model_variant":"R1T Chimera","model_version":"DeepSeek-TNG R1T","name":"DeepSeek-TNG R1T Chimera","open_source":true,"parameter_size":"671B","provider":"tng","url":""},"deepseek-tng-r1t-chimera-20250427":{"country":"DE","id":"deepseek-tng-r1t-chimera-20250427","launched":"2025-04-27","modalities":["text"],"model":"DeepSeek R1T Chimera","model_family":"DeepSeek","model_variant":"R1T Chimera","model_version":"2025-04-27","name":"DeepSeek-R1T-Chimera","open_source":true,"parameter_size":"685B","provider":"tng","url":""},"deepseek-tng-r1t2-chimera":{"country":"DE","id":"deepseek-tng-r1t2-chimera","launched":"2025-07-08","modalities":["text"],"model":"DeepSeek-TNG R1T2 Chimera","model_family":"DeepSeek-TNG","model_variant":"R1T2 Chimera","model_version":"DeepSeek-TNG R1T2","name":"DeepSeek-TNG R1T2 Chimera","open_source":true,"parameter_size":"671B","provider":"tng","url":"https://huggingface.co/tngtech/DeepSeek-TNG-R1T2-Chimera"},"deepseek-v2-chat":{"country":"CN","id":"deepseek-v2-chat","launched":"","modalities":["text"],"model":"Deepseek V2 Chat","model_family":"DeepSeek","model_variant":"Deepseek V2 Chat","model_version":"DeepSeek Coder","name":"Deepseek V2 Chat","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-v2.5":{"country":"CN","id":"deepseek-v2.5","launched":"2024-05-06","modalities":["text"],"model":"DeepSeek V2.5","model_family":"DeepSeek","model_variant":"DeepSeekV2.5","model_version":"DeepSeek V2.5","name":"DeepSeek V2.5","open_source":true,"parameter_size":"236B","provider":"deepseek","url":""},"deepseek-v2.5-1210":{"country":"CN","id":"deepseek-v2.5-1210","launched":"2024-05-06","modalities":["text"],"model":"Deepseek V2.5 1210","model_family":"DeepSeek","model_variant":"Deepseek V2.5 1210","model_version":"DeepSeek Coder","name":"Deepseek V2.5 1210","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-v2.5.1210":{"country":"CN","id":"deepseek-v2.5.1210","launched":"2024-05-06","modalities":["text"],"model":"Deepseek V2.5 1210","model_family":"DeepSeek","model_variant":"Deepseek V2.5.1210","model_version":"DeepSeek Coder","name":"Deepseek V2.5.1210","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepseek-v3":{"country":"CN","id":"deepseek-v3","launched":"2025-03-25","modalities":["text"],"model":"DeepSeek V3","model_family":"DeepSeek","model_variant":"DeepSeekV3","model_version":"DeepSeek V3","name":"DeepSeek V3","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-v3-0324":{"country":"CN","id":"deepseek-v3-0324","launched":"2025-03-25","modalities":["text"],"model":"DeepSeek V3","model_family":"DeepSeek","model_variant":"DeepSeek V3 0324","model_version":"DeepSeek V3","name":"DeepSeek V3 0324","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-v3-0528":{"country":"CN","id":"deepseek-v3-0528","launched":"2025-03-25","modalities":["text"],"model":"DeepSeek V3","model_family":"DeepSeek","model_variant":"DeepSeek V3 0528","model_version":"DeepSeek V3","name":"DeepSeek V3 0528","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-v3-base":{"country":"CN","id":"deepseek-v3-base","launched":"2025-03-25","modalities":["text"],"model":"DeepSeek V3","model_family":"DeepSeek","model_variant":"DeepSeek V3 Base","model_version":"DeepSeek V3","name":"DeepSeek V3 Base","open_source":true,"parameter_size":"671B","provider":"deepseek","url":""},"deepseek-v3.1":{"country":"CN","id":"deepseek-v3.1","launched":"2025-08-21","modalities":["text"],"model":"DeepSeek V3.1","model_family":"DeepSeek","model_variant":"DeepSeekV3.1","model_version":"DeepSeek V3.1","name":"DeepSeek V3.1","open_source":true,"parameter_size":"840B","provider":"deepseek","url":""},"deepseek-v3.1-terminus":{"country":"CN","id":"deepseek-v3.1-terminus","launched":"2025-09-22","modalities":["text"],"model":"DeepSeek V3.1 Terminus","model_family":"DeepSeek","model_variant":"DeepSeek V3.1 Terminus","model_version":"DeepSeek V3.1","name":"DeepSeek V3.1 Terminus","open_source":true,"parameter_size":"685B","provider":"deepseek","url":""},"deepseek-v3.1-terminus-thinking":{"country":"CN","id":"deepseek-v3.1-terminus-thinking","launched":"","modalities":["text"],"model":"DeepSeek V3.1 Terminus Thinking","model_family":"DeepSeek","model_variant":"DeepSeek-V3.1 Terminus Thinking","model_version":"DeepSeek V3.1","name":"DeepSeek-V3.1 Terminus Thinking","open_source":false,"parameter_size":"","provider":"deepseek","url":""},"deepseek-v3.1-thinking":{"country":"CN","id":"deepseek-v3.1-thinking","launched":"2025-08-28","modalities":["text"],"model":"DeepSeek V3.1","model_family":"DeepSeek","model_variant":"DeepSeekV3.1","model_version":"DeepSeek V3.1","name":"DeepSeek V3.1-Thinking","open_source":true,"parameter_size":"840B","provider":"deepseek","url":""},"deepseek-v3.2":{"country":"CN","id":"deepseek-v3.2","launched":"2025-09-29","modalities":["text"],"model":"DeepSeek V3.2","model_family":"DeepSeek","model_variant":"DeepSeek V3.2","model_version":"DeepSeek V3.2","name":"DeepSeek V3.2","open_source":true,"parameter_size":"685B","provider":"deepseek","url":""},"deepseek-v3.2-exp":{"country":"CN","id":"deepseek-v3.2-exp","launched":"2025-09-29","modalities":["text"],"model":"DeepSeek V3.2-Exp","model_family":"DeepSeek","model_variant":"DeepSeekV3.2-Exp","model_version":"DeepSeek V3.2","name":"DeepSeek V3.2-Exp","open_source":true,"parameter_size":"685B","provider":"deepseek","url":""},"deepseek-v3.2-exp-thinking":{"country":"CN","id":"deepseek-v3.2-exp-thinking","launched":"","modalities":["text"],"model":"DeepSeek V3.2-Exp","model_family":"DeepSeek","model_variant":"DeepSeekV3.2-Exp Thinking","model_version":"DeepSeek V3.2","name":"DeepSeek V3.2-Exp Thinking","open_source":true,"parameter_size":"685B","provider":"deepseek","url":""},"deepseek-v3.2-speciale":{"country":"CN","id":"deepseek-v3.2-speciale","launched":"2025-12-01","modalities":["text"],"model":"DeepSeek V3.2 Speciale","model_family":"DeepSeek","model_variant":"DeepSeek V3.2 Speciale","model_version":"DeepSeek V3.2","name":"DeepSeek V3.2-Speciale","open_source":true,"parameter_size":"685B","provider":"deepseek","url":""},"deepseek-v3.2-thinking":{"country":"CN","id":"deepseek-v3.2-thinking","launched":"2025-12-01","modalities":["text"],"model":"DeepSeek V3.2","model_family":"DeepSeek","model_variant":"DeepSeek V3.2","model_version":"DeepSeek V3.2","name":"DeepSeek V3.2-Thinking","open_source":true,"parameter_size":"685B","provider":"deepseek","url":""},"deepseek-vl-7b":{"country":"CN","id":"deepseek-vl-7b","launched":"","modalities":["visual"],"model":"DeepSeek VL 7B","model_family":"DeepSeek","model_variant":"VL-7B","model_version":"DeepSeek VL","name":"DeepSeek-VL-7B","open_source":true,"parameter_size":"7B","provider":"deepseek","url":""},"deepseekmath-v2-heavy":{"country":"CN","id":"deepseekmath-v2-heavy","launched":"2025-11-26","modalities":["text"],"model":"DeepSeekMath V2 Heavy","model_family":"DeepSeekMath","model_variant":"DeepSeekMath V2 Heavy","model_version":"DeepSeekMath V2","name":"DeepSeekMath-V2-Heavy","open_source":true,"parameter_size":"","provider":"deepseek","url":""},"deepswe-32b-preview":{"country":"US","id":"deepswe-32b-preview","launched":"","modalities":["text"],"model":"DeepSWE 32B","model_family":"DeepSWE","model_variant":"32B-Preview","model_version":"DeepSWE","name":"DeepSWE-32B-Preview","open_source":true,"parameter_size":"32B","provider":"deepswe","url":""},"devstral":{"country":"FR","id":"devstral","launched":"2025-05-21","modalities":["text"],"model":"Devstral","model_family":"Devstral","model_variant":"Devstral","model_version":"Devstral","name":"Devstral","open_source":true,"parameter_size":"","provider":"mistral","url":""},"devstral-medium":{"country":"FR","id":"devstral-medium","launched":"","modalities":["text"],"model":"Devstral Medium","model_family":"Devstral","model_variant":"Medium","model_version":"Devstral","name":"Devstral Medium","open_source":true,"parameter_size":"","provider":"mistral","url":""},"devstral-small-1.1":{"country":"FR","id":"devstral-small-1.1","launched":"","modalities":["text"],"model":"Devstral Small 1.1","model_family":"Devstral","model_variant":"Small 1.1","model_version":"Devstral","name":"Devstral Small 1.1","open_source":true,"parameter_size":"","provider":"mistral","url":""},"dolly-v2-12b":{"country":"US","id":"dolly-v2-12b","launched":"","modalities":["text"],"model":"Dolly v2 12B","model_family":"Dolly","model_variant":"Dolly v2 12B","model_version":"Dolly v2","name":"Dolly v2 12B","open_source":true,"parameter_size":"12B","provider":"databricks","url":""},"dolphin":{"country":"CN","id":"dolphin","launched":"","modalities":["visual"],"model":"Dolphin","model_family":"Dolphin","model_variant":"Base","model_version":"Dolphin","name":"Dolphin","open_source":true,"parameter_size":"0.3B","provider":"bytedance","published_date":"2025-05-20","url":""},"dolphin-1.5":{"country":"CN","id":"dolphin-1.5","launched":"","modalities":["visual"],"model":"Dolphin 1.5","model_family":"Dolphin","model_variant":"1.5","model_version":"Dolphin","name":"Dolphin 1.5","open_source":true,"parameter_size":"1.5B","provider":"bytedance","published_date":"2025-10-16","url":""},"dolphin-2.2.1-mistral-7b":{"country":"US","id":"dolphin-2.2.1-mistral-7b","launched":"","modalities":["text"],"model":"Dolphin Mistral 7B","model_family":"Dolphin","model_variant":"Dolphin 2.2.1 Mistral","model_version":"Dolphin","name":"Dolphin 2.2.1 Mistral 7B","open_source":true,"parameter_size":"7B","provider":"cognitivecomputations","url":""},"dolphin-2.6-mixtral-8x7b":{"country":"US","id":"dolphin-2.6-mixtral-8x7b","launched":"","modalities":["text"],"model":"dolphin-2.6-mixtral-8x7b","model_family":"dolphin-2.6-mixtral-8x7b","model_variant":"dolphin-2.6-mixtral-8x7b","model_version":"dolphin-2.6-mixtral-8x7b","name":"dolphin-2.6-mixtral-8x7b","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"dolphin-v2":{"country":"CN","id":"dolphin-v2","launched":"2025-12-01","modalities":["text"],"model":"Dolphin V2","model_family":"Dolphin","model_variant":"V2","model_version":"Dolphin V2","name":"Dolphin V2","open_source":true,"parameter_size":"4B","provider":"bytedance","url":""},"dots-ocr":{"country":"US","id":"dots-ocr","launched":"","modalities":["visual"],"model":"dots.ocr","model_family":"dots.ocr","model_variant":"Base","model_version":"dots.ocr","name":"dots.ocr","open_source":false,"parameter_size":"","provider":"dots","url":""},"dots-vlm1":{"country":"CN","id":"dots-vlm1","launched":"2025-08-05","modalities":["visual"],"model":"dots.vlm1","model_family":"dots","model_variant":"vlm1","model_version":"dots.vlm1","name":"dots.vlm1","open_source":true,"parameter_size":"672B","provider":"rednote","url":""},"dots.ocr":{"country":"CN","id":"dots.ocr","launched":"","modalities":["visual"],"model":"dots.ocr","model_family":"dots","model_variant":"ocr","model_version":"dots","name":"dots.ocr","open_source":true,"parameter_size":"3B","provider":"rednote","url":""},"doubao-1.5-vision-pro":{"country":"CN","id":"doubao-1.5-vision-pro","launched":"","modalities":["visual"],"model":"Doubao 1.5 Vision Pro","model_family":"Doubao","model_variant":"1.5 Vision Pro","model_version":"Doubao 1.5","name":"doubao-1-5-vision-pro","open_source":false,"parameter_size":"70Be","provider":"bytedance","url":""},"doubao-1.5-vision-pro-thinking":{"country":"CN","id":"doubao-1.5-vision-pro-thinking","launched":"","modalities":["text"],"model":"Doubao 1.5 Vision Pro","model_family":"Doubao","model_variant":"1.5 Vision Pro Thinking","model_version":"Doubao 1.5","name":"doubao-1-5-vision-pro Thinking","open_source":false,"parameter_size":"70Be","provider":"bytedance","url":""},"dracarys-72b-instruct":{"country":"US","id":"dracarys-72b-instruct","launched":"","modalities":["text"],"model":"Dracarys 72B","model_family":"Dracarys","model_variant":"Dracarys 72B Instruct","model_version":"Dracarys","name":"Dracarys 72B Instruct","open_source":true,"parameter_size":"72B","provider":"dracarys","url":""},"dracarys-llama-3.1-70b-instruct":{"country":"US","id":"dracarys-llama-3.1-70b-instruct","launched":"","modalities":["text"],"model":"Dracarys Llama 3.1 70B","model_family":"Dracarys","model_variant":"Dracarys Llama 3.1 70B Instruct","model_version":"Dracarys Llama 3.1","name":"Dracarys Llama 3.1 70B Instruct","open_source":true,"parameter_size":"70B","provider":"dracarys","url":""},"dream-7b-base":{"country":"CN","id":"dream-7b-base","launched":"2025-04-02","modalities":["text"],"model":"Dream 7B","model_family":"Dream","model_variant":"Dream 7B Base","model_version":"Dream","name":"Dream-7B-Base","open_source":true,"parameter_size":"7B","provider":"huawei","url":""},"dream-7b-instruct":{"country":"CN","id":"dream-7b-instruct","launched":"2025-04-02","modalities":["text"],"model":"Dream 7B","model_family":"Dream","model_variant":"Dream 7B Instruct","model_version":"Dream 7B","name":"Dream-7B-Instruct","open_source":true,"parameter_size":"7B","provider":"huawei","url":""},"eagle2.5-8b":{"country":"","id":"eagle2.5-8b","launched":"","modalities":["visual"],"model":"Eagle2.5 8B","model_family":"Eagle","model_variant":"Eagle2.5-8B","model_version":"Eagle 2.5","name":"Eagle2.5-8B","open_source":true,"parameter_size":"8B","provider":"unknown","url":""},"embeddinggemma-300m":{"country":"US","id":"embeddinggemma-300m","launched":"","modalities":["text"],"model":"EmbeddingGemma 300M","model_family":"Embeddinggemma","model_variant":"Embeddinggemma","model_version":"Embeddinggemma","name":"EmbeddingGemma 300M","open_source":true,"parameter_size":"0.3B","provider":"google","url":""},"emu3-8b":{"country":"CN","id":"emu3-8b","launched":"","modalities":["visual"],"model":"Emu3 8B","model_family":"Emu","model_variant":"Emu3 8B","model_version":"Emu3","name":"Emu3 8B","open_source":true,"parameter_size":"8B","provider":"baai","url":""},"ernie-4.5":{"country":"CN","id":"ernie-4.5","launched":"2025-06-30","modalities":["visual"],"model":"ERNIE 4.5 424B","model_family":"ERNIE","model_variant":"ERNIE 4.5","model_version":"ERNIE 4.5","name":"ERNIE 4.5 424B A47B","open_source":true,"parameter_size":"424B","provider":"baidu","url":""},"ernie-4.5-21b-a3b-pt":{"country":"CN","id":"ernie-4.5-21b-a3b-pt","launched":"2025-06-30","modalities":["text"],"model":"ERNIE 4.5 21B A3B","model_family":"ERNIE","model_variant":"4.5-21B-A3B-PT","model_version":"ERNIE 4.5","name":"ERNIE 4.5 21B A3B PT","open_source":true,"parameter_size":"21B","provider":"baidu","url":""},"ernie-4.5-vl-28b-a3b-thinking":{"country":"CN","id":"ernie-4.5-vl-28b-a3b-thinking","launched":"2025-11-10","modalities":["visual"],"model":"ERNIE 4.5 VL 28B A3B Thinking","model_family":"ERNIE","model_variant":"4.5 VL 28B A3B Thinking","model_version":"ERNIE 4.5 VL","name":"ERNIE-4.5-VL-28B-A3B-Thinking","open_source":true,"parameter_size":"28B","provider":"baidu","url":""},"ernie-4.5-vl-424b-a47b":{"country":"CN","id":"ernie-4.5-vl-424b-a47b","launched":"2025-06-30","modalities":["visual"],"model":"ERNIE-4.5 VL424B A47B","model_family":"ERNIE","model_variant":"4.5 VL424B-A47B","model_version":"ERNIE 4.5 VL","name":"ERNIE 4.5 VL424B A47B","open_source":true,"parameter_size":"424B","provider":"baidu","url":""},"ernie-x1":{"country":"CN","id":"ernie-x1","launched":"","modalities":["text"],"model":"ERNIE X1","model_family":"ERNIE","model_variant":"X1","model_version":"ERNIE X1","name":"ERNIE X1","open_source":true,"parameter_size":"","provider":"baidu","url":""},"eurollm-22b":{"country":"EU","id":"eurollm-22b","launched":"2025-12-15","modalities":["text"],"model":"EuroLLM 22B","model_family":"EuroLLM","model_variant":"EuroLLM-22B","model_version":"EuroLLM","name":"EuroLLM-22B","open_source":"plus","parameter_size":"22B","provider":"eurollm","url":""},"evabyte-6.5b":{"country":"CN","id":"evabyte-6.5b","launched":"2025-01-20","modalities":["text"],"model":"EvaByte 6.5B","model_family":"EvaByte","model_variant":"EvaByte 6.5B","model_version":"EvaByte 6.5B","name":"EvaByte 6.5B","open_source":true,"parameter_size":"6.5B","provider":"hku","url":""},"exaone-3.5-2.4b":{"country":"KR","id":"exaone-3.5-2.4b","launched":"2024-12-09","modalities":["text"],"model":"EXAONE 3.5 2.4B","model_family":"EXAONE","model_variant":"3.5-2.4B","model_version":"EXAONE 3.5","name":"EXAONE 3.5 2.4B","open_source":true,"parameter_size":"2.4B","provider":"lg","url":""},"exaone-3.5-2.4b-instruct":{"country":"KR","id":"exaone-3.5-2.4b-instruct","launched":"2024-12-09","modalities":["text"],"model":"EXAONE 3.5 2.4B","model_family":"EXAONE","model_variant":"3.5-2.4B Instruct","model_version":"EXAONE 3.5","name":"EXAONE 3.5 2.4B Instruct","open_source":true,"parameter_size":"2.4B","provider":"lg","url":""},"exaone-3.5-7.8b-instruct":{"country":"KR","id":"exaone-3.5-7.8b-instruct","launched":"","modalities":["text"],"model":"EXAONE 3.5 7.8B","model_family":"EXAONE","model_variant":"3.5-7.8B Instruct","model_version":"EXAONE 3.5","name":"EXAONE 3.5 7.8B Instruct","open_source":true,"parameter_size":"7.8B","provider":"lg","url":""},"exaone-4.0-1.2b":{"country":"KR","id":"exaone-4.0-1.2b","launched":"2025-07-15","modalities":["text"],"model":"EXAONE 4.0 1.2B","model_family":"EXAONE","model_variant":"4.0 1.2B","model_version":"EXAONE 4.0","name":"EXAONE 4.0 1.2B","open_source":true,"parameter_size":"1.2B","provider":"lg","url":""},"exaone-4.0-1.2b-thinking":{"country":"KR","id":"exaone-4.0-1.2b-thinking","launched":"","modalities":["text"],"model":"EXAONE 4.0 1.2B","model_family":"EXAONE","model_variant":"4.0 1.2B Thinking","model_version":"EXAONE 4.0","name":"EXAONE 4.0 1.2B (Thinking)","open_source":true,"parameter_size":"1.2B","provider":"lg","url":""},"exaone-4.0-32b":{"country":"KR","id":"exaone-4.0-32b","launched":"2025-06-15","modalities":["text"],"model":"EXAONE 4.0 32B","model_family":"EXAONE","model_variant":"4.0-32B","model_version":"EXAONE 4.0","name":"EXAONE 4.0 32B","open_source":true,"parameter_size":"32B","provider":"lg","url":""},"exaone-4.0-32b-thinking":{"country":"KR","id":"exaone-4.0-32b-thinking","launched":"","modalities":["text"],"model":"EXAONE 4.0 32B","model_family":"EXAONE","model_variant":"4.0-32B Thinking","model_version":"EXAONE 4.0","name":"EXAONE 4.0 32B Thinking","open_source":true,"parameter_size":"32B","provider":"lg","url":""},"exaone-deep-2.4b":{"country":"KR","id":"exaone-deep-2.4b","launched":"2025-03-18","modalities":["text"],"model":"EXAONE Deep 2.4B","model_family":"EXAONE","model_variant":"Deep 2.4B","model_version":"EXAONE Deep","name":"EXAONE Deep 2.4B","open_source":true,"parameter_size":"2.4B","provider":"lg","url":""},"falcon-180b-chat":{"country":"AE","id":"falcon-180b-chat","launched":"","modalities":["text"],"model":"Falcon 180B","model_family":"Falcon","model_variant":"Falcon 180B Chat","model_version":"Falcon","name":"Falcon 180B Chat","open_source":true,"parameter_size":"180B","provider":"tii","url":""},"falcon-3-1b-base":{"country":"AE","id":"falcon-3-1b-base","launched":"2024-12-17","modalities":["text"],"model":"Falcon3 1B","model_family":"Falcon","model_variant":"Falcon-3-1B-Base","model_version":"Falcon 3","name":"Falcon-3-1B-Base","open_source":true,"parameter_size":"1B","provider":"tii","url":""},"falcon-40b-instruct":{"country":"AE","id":"falcon-40b-instruct","launched":"","modalities":["text"],"model":"Falcon 40B Instruct","model_family":"Falcon","model_variant":"Falcon 40B Instruct","model_version":"Falcon","name":"Falcon 40B Instruct","open_source":true,"parameter_size":"40B","provider":"tii","url":""},"falcon-7b-instruct":{"country":"AE","id":"falcon-7b-instruct","launched":"","modalities":["text"],"model":"Falcon 7B Instruct","model_family":"Falcon","model_variant":"Falcon 7B Instruct","model_version":"Falcon","name":"Falcon 7B Instruct","open_source":true,"parameter_size":"7B","provider":"tii","url":""},"falcon-e-1b-base":{"country":"AE","id":"falcon-e-1b-base","launched":"2025-05-25","modalities":["text"],"model":"Falcon E 1B","model_family":"Falcon","model_variant":"Falcon-E-1B-Base","model_version":"Falcon E","name":"Falcon-E-1B-Base","open_source":true,"parameter_size":"1B","provider":"tii","url":""},"falcon-h1-0.5b-instruct":{"country":"AE","id":"falcon-h1-0.5b-instruct","launched":"2025-05-21","modalities":["text"],"model":"Falcon H1 0.5B","model_family":"Falcon","model_variant":"H1-0.5B-Instruct","model_version":"Falcon H1","name":"Falcon-H1-0.5B-Instruct","open_source":true,"parameter_size":"0.5B","provider":"tii","url":""},"falcon-h1-1.5b":{"country":"AE","id":"falcon-h1-1.5b","launched":"2025-05-21","modalities":["text"],"model":"Falcon H1 1.5B","model_family":"Falcon","model_variant":"H1-1.5B","model_version":"Falcon H1","name":"Falcon-H1-1.5B","open_source":true,"parameter_size":"1.5B","provider":"tii","url":""},"falcon-h1-1.5b-deep":{"country":"AE","id":"falcon-h1-1.5b-deep","launched":"2025-05-21","modalities":["text"],"model":"Falcon H1 1.5B","model_family":"Falcon","model_variant":"H1-1.5B-Deep","model_version":"Falcon H1","name":"Falcon-H1-1.5B-Deep","open_source":true,"parameter_size":"1.5B","provider":"tii","url":""},"falcon-h1-34b-instruct":{"country":"AE","id":"falcon-h1-34b-instruct","launched":"2025-05-21","modalities":["text"],"model":"Falcon H1 34B","model_family":"Falcon","model_variant":"H1-34B-Instruct","model_version":"Falcon H1","name":"Falcon-H1-34B-Instruct","open_source":true,"parameter_size":"34B","provider":"tii","url":""},"falcon-h1-3b":{"country":"AE","id":"falcon-h1-3b","launched":"2025-05-21","modalities":["text"],"model":"Falcon H1 3B","model_family":"Falcon","model_variant":"H1-3B","model_version":"Falcon H1","name":"Falcon-H1-3B","open_source":true,"parameter_size":"3B","provider":"tii","url":""},"falcon-h1-7b":{"country":"AE","id":"falcon-h1-7b","launched":"2025-05-21","modalities":["text"],"model":"Falcon H1 7B","model_family":"Falcon","model_variant":"H1-7B","model_version":"Falcon H1","name":"Falcon-H1-7B","open_source":true,"parameter_size":"7B","provider":"tii","url":""},"falcon-h1r-7b":{"country":"AE","id":"falcon-h1r-7b","launched":"2026-01-05","modalities":["text"],"model":"Falcon H1R 7B","model_family":"Falcon","model_variant":"H1R-7B","model_version":"Falcon H1R","name":"Falcon-H1R-7B","open_source":true,"parameter_size":"7B","provider":"tii","url":""},"falcon3-10b":{"country":"AE","id":"falcon3-10b","launched":"2024-12-17","modalities":["text"],"model":"Falcon3 10B","model_family":"Falcon","model_variant":"Falcon3-10B","model_version":"Falcon 3","name":"Falcon3-10B","open_source":true,"parameter_size":"10B","provider":"tii","url":""},"falcon3-1b":{"country":"AE","id":"falcon3-1b","launched":"2024-12-17","modalities":["text"],"model":"Falcon3 1B","model_family":"Falcon","model_variant":"Falcon3-1B","model_version":"Falcon 3","name":"Falcon3-1B","open_source":true,"parameter_size":"1B","provider":"tii","url":""},"falcon3-3b":{"country":"AE","id":"falcon3-3b","launched":"2024-12-17","modalities":["text"],"model":"Falcon3 3B","model_family":"Falcon","model_variant":"Falcon3-3B","model_version":"Falcon 3","name":"Falcon3-3B","open_source":true,"parameter_size":"3B","provider":"tii","url":""},"falcon3-7b":{"country":"AE","id":"falcon3-7b","launched":"2024-12-17","modalities":["text"],"model":"Falcon3 7B","model_family":"Falcon","model_variant":"Falcon3-7B","model_version":"Falcon 3","name":"Falcon3-7B","open_source":true,"parameter_size":"7B","provider":"tii","url":""},"fastchat-t5-3b":{"country":"US","id":"fastchat-t5-3b","launched":"","modalities":["text"],"model":"FastChat T5 3B","model_family":"FastChat","model_variant":"FastChat T5 3B","model_version":"FastChat","name":"FastChat T5 3B","open_source":true,"parameter_size":"3B","provider":"lmsys","url":""},"fastvlm-0.5b":{"country":"US","id":"fastvlm-0.5b","launched":"","modalities":["visual"],"model":"FastVLM 0.5B","model_family":"FastVLM","model_variant":"0.5B","model_version":"FastVLM","name":"FastVLM-0.5B","open_source":true,"parameter_size":"0.5B","provider":"apple","url":"https://huggingface.co/apple/FastVLM-0.5B"},"fastvlm-1.5b":{"country":"US","id":"fastvlm-1.5b","launched":"","modalities":["visual"],"model":"FastVLM 1.5B","model_family":"FastVLM","model_variant":"1.5B","model_version":"FastVLM","name":"FastVLM-1.5B","open_source":true,"parameter_size":"1.5B","provider":"apple","url":""},"fastvlm-7b":{"country":"US","id":"fastvlm-7b","launched":"","modalities":["visual"],"model":"FastVLM 7B","model_family":"FastVLM","model_variant":"7B","model_version":"FastVLM","name":"FastVLM-7B","open_source":true,"parameter_size":"7B","provider":"apple","url":""},"flan-t5":{"country":"US","id":"flan-t5","launched":"2022-10-20","modalities":["text"],"model":"FLAN-T5","model_family":"FLAN-T5","model_variant":"T5","model_version":"FLAN-T5","name":"FLAN-T5","open_source":true,"parameter_size":"","provider":"google","url":""},"flan-ul2":{"country":"US","id":"flan-ul2","launched":"2023-03-03","modalities":["text"],"model":"FLAN-UL2","model_family":"FLAN-UL2","model_variant":"UL2","model_version":"FLAN-UL2","name":"FLAN-UL2","open_source":true,"parameter_size":"20B","provider":"google","url":""},"flux-1-dev":{"country":"DE","id":"flux-1-dev","launched":"2024-08-02","modalities":["visual"],"model":"FLUX.1","model_family":"FLUX.1","model_variant":"Dev","model_version":"FLUX.1","name":"FLUX.1 [Dev]","open_source":true,"parameter_size":"","provider":"black-forest-labs","url":""},"frogboss-32b-2510":{"country":"US","id":"frogboss-32b-2510","launched":"2026-01-12","modalities":["text"],"model":"FrogBoss 32B","model_family":"FrogBoss","model_variant":"32B-2510","model_version":"FrogBoss","name":"FrogBoss-32B-2510","open_source":true,"parameter_size":"32B","provider":"microsoft","url":""},"frogmini-14b-2510":{"country":"US","id":"frogmini-14b-2510","launched":"2026-01-12","modalities":["text"],"model":"FrogMini 14B","model_family":"FrogMini","model_variant":"14B-2510","model_version":"FrogMini","name":"FrogMini-14B-2510","open_source":true,"parameter_size":"14B","provider":"microsoft","url":""},"functiongemma":{"country":"US","id":"functiongemma","launched":"","modalities":["text"],"model":"FunctionGemma","model_family":"Gemma","model_variant":"FunctionGemma","model_version":"FunctionGemma","name":"FunctionGemma","open_source":true,"parameter_size":"","provider":"meetkai","url":""},"gemini-1.0-pro":{"country":"US","id":"gemini-1.0-pro","launched":"","modalities":["visual"],"model":"Gemini 1.0 Pro","model_family":"Gemini","model_variant":"Gemini1.0","model_version":"Gemini 1.0","name":"Gemini 1.0 Pro","open_source":false,"parameter_size":"20Be","provider":"google","url":""},"gemini-1.5-flash":{"country":"US","id":"gemini-1.5-flash","launched":"2024-05-24","modalities":["text","visual"],"model":"Gemini 1.5 Flash","model_family":"Gemini","model_variant":"Gemini 1.5 Flash","model_version":"Gemini 1.5","name":"Gemini 1.5 Flash","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-flash-001":{"country":"US","id":"gemini-1.5-flash-001","launched":"2024-05-24","modalities":["text"],"model":"Gemini 1.5 Flash","model_family":"Gemini","model_variant":"1.5 Flash 001","model_version":"Gemini 1.5","name":"Gemini 1.5 Flash 001","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-flash-002":{"country":"US","id":"gemini-1.5-flash-002","launched":"2024-05-24","modalities":["text","visual"],"model":"Gemini 1.5 Flash","model_family":"Gemini","model_variant":"Gemini 1.5 Flash 002","model_version":"Gemini 1.5","name":"Gemini 1.5 Flash (002)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-flash-8b-001":{"country":"US","id":"gemini-1.5-flash-8b-001","launched":"2024-05-24","modalities":["text","visual"],"model":"Gemini 1.5 Flash","model_family":"Gemini","model_variant":"Gemini 1.5 Flash 8B","model_version":"Gemini 1.5","name":"Gemini 1.5 Flash 8B (001)","open_source":false,"parameter_size":"8B","provider":"google","url":""},"gemini-1.5-flash-8b-0827":{"country":"US","id":"gemini-1.5-flash-8b-0827","launched":"2024-05-24","modalities":["text","visual"],"model":"Gemini 1.5 Flash","model_family":"Gemini","model_variant":"Gemini 1.5 Flash 8B 0827","model_version":"Gemini 1.5","name":"Gemini 1.5 Flash 8B (0827)","open_source":false,"parameter_size":"8B","provider":"google","url":""},"gemini-1.5-flash-api-0514":{"country":"US","id":"gemini-1.5-flash-api-0514","launched":"2024-05-24","modalities":["text"],"model":"Gemini 1.5 Flash API 0514","model_family":"Gemini","model_variant":"Gemini 1.5 Flash API 0514","model_version":"Gemini 1.0","name":"Gemini 1.5 Flash API 0514","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-flash-exp-0827":{"country":"US","id":"gemini-1.5-flash-exp-0827","launched":"2024-05-24","modalities":["text"],"model":"Gemini 1.5 Flash EXP 0827","model_family":"Gemini","model_variant":"Gemini 1.5 Flash EXP 0827","model_version":"Gemini 1.0","name":"Gemini 1.5 Flash EXP 0827","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-pro":{"country":"US","id":"gemini-1.5-pro","launched":"2024-09-24","modalities":["text","visual"],"model":"Gemini 1.5 Pro","model_family":"Gemini","model_variant":"Gemini1.5","model_version":"Gemini 1.5","name":"Gemini 1.5 Pro","open_source":false,"parameter_size":"200Be","provider":"google","url":""},"gemini-1.5-pro-001":{"country":"US","id":"gemini-1.5-pro-001","launched":"2024-09-24","modalities":["text"],"model":"Gemini 1.5 Pro","model_family":"Gemini","model_variant":"1.5 Pro 001","model_version":"Gemini 1.5","name":"Gemini 1.5 Pro 001","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-pro-002":{"country":"US","id":"gemini-1.5-pro-002","launched":"2024-09-24","modalities":["text","visual"],"model":"Gemini 1.5 Pro","model_family":"Gemini","model_variant":"Gemini 1.5 Pro 002","model_version":"Gemini 1.5","name":"Gemini 1.5 Pro (002)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-pro-api-0514":{"country":"US","id":"gemini-1.5-pro-api-0514","launched":"2024-09-24","modalities":["text"],"model":"Gemini 1.5 Pro","model_family":"Gemini","model_variant":"Gemini 1.5 Pro API 0514","model_version":"Gemini 1.5","name":"Gemini-1.5-Pro-API-0514","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-pro-exp-0801":{"country":"US","id":"gemini-1.5-pro-exp-0801","launched":"2024-09-24","modalities":["text"],"model":"Gemini 1.5 PRO EXP 0801","model_family":"Gemini","model_variant":"Gemini 1.5 PRO EXP 0801","model_version":"Gemini 1.0","name":"Gemini 1.5 PRO EXP 0801","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-1.5-pro-exp-0827":{"country":"US","id":"gemini-1.5-pro-exp-0827","launched":"2024-09-24","modalities":["text"],"model":"Gemini 1.5 PRO EXP 0827","model_family":"Gemini","model_variant":"Gemini 1.5 PRO EXP 0827","model_version":"Gemini 1.0","name":"Gemini 1.5 PRO EXP 0827","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash":{"country":"US","id":"gemini-2.0-flash","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash","model_family":"Gemini","model_variant":"Gemini2.0","model_version":"Gemini 2.0","name":"Gemini 2.0 Flash","open_source":false,"parameter_size":"50Be","provider":"google","url":""},"gemini-2.0-flash-001":{"country":"US","id":"gemini-2.0-flash-001","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash","model_family":"Gemini","model_variant":"2.0-Flash-001","model_version":"Gemini 2.0","name":"Google Gemini-2.0-Flash-001","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash-exp":{"country":"US","id":"gemini-2.0-flash-exp","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash","model_family":"Gemini","model_variant":"Gemini 2.0 Flash Exp","model_version":"Gemini 2.0","name":"Gemini-2.0-Flash-Exp","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash-lite":{"country":"US","id":"gemini-2.0-flash-lite","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash Lite","model_family":"Gemini","model_variant":"Gemini 2.0 Flash Lite","model_version":"Gemini 2.0","name":"Gemini 2.0 Flash Lite","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash-lite-preview":{"country":"US","id":"gemini-2.0-flash-lite-preview","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash Lite","model_family":"Gemini","model_variant":"2.0-Flash-Lite-Preview","model_version":"Gemini 2.0","name":"Google Gemini-2.0-Flash-Lite-Preview","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash-lite-preview-02-05":{"country":"US","id":"gemini-2.0-flash-lite-preview-02-05","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash","model_family":"Gemini","model_variant":"Gemini 2.0 Flash Lite Preview 02 05","model_version":"Gemini 2.0","name":"Gemini-2.0-Flash-Lite-Preview-02-05","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash-thinking-01-21":{"country":"US","id":"gemini-2.0-flash-thinking-01-21","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash","model_family":"Gemini","model_variant":"Gemini 2.0 Flash Thinking","model_version":"Gemini 2.0","name":"Gemini 2.0 Flash Thinking (01-21)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash-thinking-exp-01-21":{"country":"US","id":"gemini-2.0-flash-thinking-exp-01-21","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash","model_family":"Gemini","model_variant":"Gemini 2.0 Flash Thinking Exp 01 21","model_version":"Gemini 2.0","name":"Gemini-2.0-Flash-Thinking-Exp-01-21","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-flash-thinking-exp-1219":{"country":"US","id":"gemini-2.0-flash-thinking-exp-1219","launched":"2025-01-30","modalities":["text"],"model":"Gemini 2.0 Flash","model_family":"Gemini","model_variant":"Gemini 2.0 Flash Thinking Exp 1219","model_version":"Gemini 2.0","name":"Gemini-2.0-Flash-Thinking-Exp-1219","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-pro":{"country":"US","id":"gemini-2.0-pro","launched":"2025-02-05","modalities":["text"],"model":"Gemini 2.0 Pro","model_family":"Gemini","model_variant":"Gemini2.0","model_version":"Gemini 2.0","name":"Gemini 2.0 Pro","open_source":false,"parameter_size":"500Be","provider":"google","url":""},"gemini-2.0-pro-exp":{"country":"US","id":"gemini-2.0-pro-exp","launched":"2025-02-05","modalities":["text"],"model":"Gemini 2.0 Pro","model_family":"Gemini","model_variant":"2.0-Pro-Exp","model_version":"Gemini 2.0","name":"Google Gemini-2.0-Pro-Exp","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.0-pro-exp-02-05":{"country":"US","id":"gemini-2.0-pro-exp-02-05","launched":"2025-02-05","modalities":["text"],"model":"Gemini 2.0 Pro","model_family":"Gemini","model_variant":"Gemini 2.0 Pro Exp 02 05","model_version":"Gemini 2.0","name":"Gemini-2.0-Pro-Exp-02-05","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5":{"country":"US","id":"gemini-2.5","launched":"","modalities":["text","visual"],"model":"Gemini 2.5","model_family":"Gemini","model_variant":"Gemini 2.5","model_version":"Gemini 2.5","name":"Gemini 2.5","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-deep-think":{"country":"US","id":"gemini-2.5-deep-think","launched":"2025-08-20","modalities":["text"],"model":"Gemini 2.5 Deep Think","model_family":"Gemini","model_variant":"2.5 Deep Think","model_version":"Gemini 2.5","name":"Gemini 2.5 Deep Think","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash":{"country":"US","id":"gemini-2.5-flash","launched":"2025-04-17","modalities":["visual"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini2.5","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash","open_source":false,"parameter_size":"70Be","provider":"google","url":""},"gemini-2.5-flash-04-17":{"country":"US","id":"gemini-2.5-flash-04-17","launched":"2025-04-17","modalities":["visual"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash 04-17","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash (04-17)","open_source":false,"parameter_size":"70Be","provider":"google","url":""},"gemini-2.5-flash-05-20":{"country":"US","id":"gemini-2.5-flash-05-20","launched":"2025-04-17","modalities":["visual"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash 05-20","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash (05-20)","open_source":false,"parameter_size":"70Be","provider":"google","url":""},"gemini-2.5-flash-2024-05":{"country":"US","id":"gemini-2.5-flash-2024-05","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"2.5 Flash 2024-05","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash (2024-05)","open_source":false,"parameter_size":"70Be","provider":"google","url":""},"gemini-2.5-flash-lite":{"country":"US","id":"gemini-2.5-flash-lite","launched":"2025-06-17","modalities":["visual"],"model":"Gemini 2.5 Flash Lite","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Lite","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Lite","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-lite-preview-09-2025":{"country":"US","id":"gemini-2.5-flash-lite-preview-09-2025","launched":"2025-06-17","modalities":["text"],"model":"Gemini 2.5 Flash Lite","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Lite Preview","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Lite Preview (09-2025)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-lite-preview-thinking":{"country":"US","id":"gemini-2.5-flash-lite-preview-thinking","launched":"2025-06-17","modalities":["text"],"model":"Gemini 2.5 Flash Lite (Thinking)","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Lite Preview Thinking","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Lite Preview (Thinking)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-04-17":{"country":"US","id":"gemini-2.5-flash-preview-04-17","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"2.5 Flash Preview 04-17","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (04-17)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-05-20":{"country":"US","id":"gemini-2.5-flash-preview-05-20","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"2.5 Flash Preview 05-20","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (05-20)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-05-20-thinking":{"country":"US","id":"gemini-2.5-flash-preview-05-20-thinking","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"2.5 Flash Preview Thinking","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (Thinking)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-09-2025":{"country":"US","id":"gemini-2.5-flash-preview-09-2025","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Preview","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (09-2025)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-thinking-16k":{"country":"US","id":"gemini-2.5-flash-preview-thinking-16k","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Preview Thinking 16K","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (Thinking 16k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-thinking-1k":{"country":"US","id":"gemini-2.5-flash-preview-thinking-1k","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Preview Thinking 1K","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (Thinking 1k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-thinking-24k":{"country":"US","id":"gemini-2.5-flash-preview-thinking-24k","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Preview Thinking 24K","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (Thinking 24k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-preview-thinking-8k":{"country":"US","id":"gemini-2.5-flash-preview-thinking-8k","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Preview Thinking 8K","model_version":"Gemini 2.5","name":"Gemini 2.5 Flash Preview (Thinking 8k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-flash-thinking":{"country":"US","id":"gemini-2.5-flash-thinking","launched":"2025-04-17","modalities":["text"],"model":"Gemini 2.5 Flash","model_family":"Gemini","model_variant":"Gemini 2.5 Flash Thinking","model_version":"Gemini 2.5","name":"Gemini-2.5-Flash Thinking","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro":{"country":"US","id":"gemini-2.5-pro","launched":"2025-03-25","modalities":["text","visual"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini2.5","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro","open_source":false,"parameter_size":"700Be","provider":"google","url":""},"gemini-2.5-pro-05-06":{"country":"US","id":"gemini-2.5-pro-05-06","launched":"2025-03-25","modalities":["text","visual"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro 05-06","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro (05-06)","open_source":false,"parameter_size":"700Be","provider":"google","url":""},"gemini-2.5-pro-06-05":{"country":"US","id":"gemini-2.5-pro-06-05","launched":"2025-03-25","modalities":["text","visual"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro 06-05","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro (06-05)","open_source":false,"parameter_size":"700Be","provider":"google","url":""},"gemini-2.5-pro-exp-03.25":{"country":"US","id":"gemini-2.5-pro-exp-03.25","launched":"2025-03-25","modalities":["text"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 PRO EXP 03.25","model_version":"Gemini 2.5","name":"Gemini 2.5 PRO EXP 03.25","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-max-thinking":{"country":"US","id":"gemini-2.5-pro-max-thinking","launched":"2025-03-25","modalities":["text"],"model":"Gemini 2.5 Pro (Max Thinking)","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Max Thinking","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro (Max Thinking)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-preview":{"country":"US","id":"gemini-2.5-pro-preview","launched":"2025-03-25","modalities":["text","visual"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Preview","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro Preview","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-preview-03-25":{"country":"US","id":"gemini-2.5-pro-preview-03-25","launched":"2025-03-25","modalities":["text","visual"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"2.5 Pro Preview 03-25","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro Preview (03-25)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-preview-05-06":{"country":"US","id":"gemini-2.5-pro-preview-05-06","launched":"2025-03-25","modalities":["text","visual"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"2.5 Pro Preview 05-06","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro Preview (05-06)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-preview-06-05":{"country":"US","id":"gemini-2.5-pro-preview-06-05","launched":"2025-03-25","modalities":["text","visual"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Preview 06-05","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro Preview (06-05)","open_source":false,"parameter_size":"700Be","provider":"google","url":""},"gemini-2.5-pro-preview-thinking-1k":{"country":"US","id":"gemini-2.5-pro-preview-thinking-1k","launched":"2025-03-25","modalities":["text"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Preview Thinking 1K","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro Preview (Thinking 1k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-thinking-16k":{"country":"US","id":"gemini-2.5-pro-thinking-16k","launched":"2025-03-25","modalities":["text"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Thinking 16K","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro (Thinking 16k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-thinking-1k":{"country":"US","id":"gemini-2.5-pro-thinking-1k","launched":"2025-03-25","modalities":["text"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Thinking 1K","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro (Thinking 1k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-thinking-32k":{"country":"US","id":"gemini-2.5-pro-thinking-32k","launched":"2025-03-25","modalities":["text"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Thinking 32K","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro (Thinking 32k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-2.5-pro-thinking-8k":{"country":"US","id":"gemini-2.5-pro-thinking-8k","launched":"2025-03-25","modalities":["text"],"model":"Gemini 2.5 Pro","model_family":"Gemini","model_variant":"Gemini 2.5 Pro Thinking 8K","model_version":"Gemini 2.5","name":"Gemini 2.5 Pro (Thinking 8k)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-3-deep-think":{"country":"US","id":"gemini-3-deep-think","launched":"2026-02-12","modalities":["text"],"model":"Gemini 3 Deep Think","model_family":"Gemini","model_variant":"Gemini 3 Deep Think","model_version":"Gemini 3","name":"Gemini 3 Deep Think","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-3-flash":{"country":"US","id":"gemini-3-flash","launched":"2025-12-17","modalities":["text","visual"],"model":"Gemini 3 Flash","model_family":"Gemini","model_variant":"Gemini 3 Flash","model_version":"Gemini 3","name":"Gemini 3 Flash","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-3-flash-preview":{"country":"US","id":"gemini-3-flash-preview","launched":"","modalities":["text","visual"],"model":"Gemini 3 Flash Preview","model_family":"Gemini","model_variant":"Gemini 3 Flash Preview","model_version":"Gemini 3","name":"Gemini 3 Flash Preview","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-3-pro":{"country":"US","id":"gemini-3-pro","launched":"2025-11-18","modalities":["text","visual"],"model":"Gemini 3 Pro","model_family":"Gemini","model_variant":"Gemini 3","model_version":"Gemini 3","name":"Gemini 3 Pro","open_source":false,"parameter_size":"3000Be","provider":"google","url":""},"gemini-3-pro-preview":{"country":"US","id":"gemini-3-pro-preview","launched":"2025-11-18","modalities":["text","visual"],"model":"Gemini 3 Pro Preview","model_family":"Gemini","model_variant":"Gemini 3 Preview","model_version":"Gemini 3","name":"Gemini 3 Pro Preview","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-3.1-pro":{"country":"US","id":"gemini-3.1-pro","launched":"2026-02-19","modalities":["text","visual"],"model":"Gemini 3.1 Pro","model_family":"Gemini","model_variant":"Gemini 3.1","model_version":"Gemini 3.1","name":"Gemini 3.1 Pro","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-advanced-0514":{"country":"US","id":"gemini-advanced-0514","launched":"2025-05-14","modalities":["text"],"model":"Gemini Advanced","model_family":"Gemini","model_variant":"Gemini Advanced","model_version":"Gemini","name":"Gemini Advanced (05-14)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-deep-think-imo-gold":{"country":"US","id":"gemini-deep-think-imo-gold","launched":"2025-09-17","modalities":["text"],"model":"Gemini Deep Think IMO Gold","model_family":"Gemini","model_variant":"Deep Think IMO Gold","model_version":"Gemini Deep Think","name":"Gemini Deep Think (IMO Gold)","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-deepresearch":{"country":"US","id":"gemini-deepresearch","launched":"2024-12-11","modalities":["text"],"model":"Gemini DeepResearch","model_family":"Gemini","model_variant":"Gemini DeepResearch","model_version":"Gemini DeepResearch","name":"Gemini DeepResearch","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-diffusion":{"country":"US","id":"gemini-diffusion","launched":"","modalities":["text","visual"],"model":"Gemini Diffusion","model_family":"Gemini","model_variant":"Gemini Diffusion","model_version":"Gemini Diffusion","name":"Gemini Diffusion","open_source":false,"parameter_size":"20Be","provider":"google","url":""},"gemini-exp-1114":{"country":"US","id":"gemini-exp-1114","launched":"","modalities":["text"],"model":"Gemini EXP 1114","model_family":"Gemini","model_variant":"Gemini EXP 1114","model_version":"Gemini 1.0","name":"Gemini EXP 1114","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-exp-1121":{"country":"US","id":"gemini-exp-1121","launched":"","modalities":["text"],"model":"Gemini EXP 1121","model_family":"Gemini","model_variant":"Gemini EXP 1121","model_version":"Gemini 1.0","name":"Gemini EXP 1121","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-exp-1206":{"country":"US","id":"gemini-exp-1206","launched":"","modalities":["text"],"model":"Gemini EXP 1206","model_family":"Gemini","model_variant":"Gemini EXP 1206","model_version":"Gemini 1.0","name":"Gemini EXP 1206","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-nano-2":{"country":"US","id":"gemini-nano-2","launched":"2023-12-06","modalities":["visual"],"model":"Gemini Nano 2","model_family":"Gemini","model_variant":"Gemini Nano 2","model_version":"Gemini Nano","name":"Gemini Nano 2","open_source":false,"parameter_size":"3.8B","provider":"google","url":""},"gemini-pro":{"country":"US","id":"gemini-pro","launched":"","modalities":["text"],"model":"Gemini Pro","model_family":"Gemini","model_variant":"Gemini Pro","model_version":"Gemini","name":"Gemini Pro","open_source":false,"parameter_size":"","provider":"google","url":""},"gemini-pro-dev-api":{"country":"US","id":"gemini-pro-dev-api","launched":"","modalities":["text"],"model":"Gemini Pro","model_family":"Gemini","model_variant":"Gemini Pro Dev API","model_version":"Gemini","name":"Gemini Pro Dev API","open_source":false,"parameter_size":"","provider":"google","url":""},"gemma-1.1-2b-it":{"country":"US","id":"gemma-1.1-2b-it","launched":"","modalities":["text"],"model":"gemma-1.1-2b","model_family":"gemma-1.1-2b-it","model_variant":"gemma-1.1-2b-it","model_version":"gemma-1.1","name":"gemma-1.1-2b-it","open_source":true,"parameter_size":"2B","provider":"google","url":""},"gemma-1.1-7b-it":{"country":"US","id":"gemma-1.1-7b-it","launched":"2024-02-21","modalities":["text"],"model":"gemma-1.1-7b","model_family":"Gemma","model_variant":"gemma-1.1-7b-it","model_version":"gemma-1.1","name":"gemma-1.1-7b-it","open_source":true,"parameter_size":"7B","provider":"google","url":""},"gemma-2-27b":{"country":"US","id":"gemma-2-27b","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 27B","model_family":"Gemma","model_variant":"Gemma 2","model_version":"Gemma 2","name":"Gemma 2 27B","open_source":true,"parameter_size":"27B","provider":"google","url":""},"gemma-2-27b-instruct":{"country":"US","id":"gemma-2-27b-instruct","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2.27B","model_family":"Gemma","model_variant":"Gemma 2.27B Instruct","model_version":"Gemma 2","name":"Gemma 2.27B Instruct","open_source":true,"parameter_size":"2.27B","provider":"google","url":""},"gemma-2-27b-it":{"country":"US","id":"gemma-2-27b-it","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 27B IT","model_family":"Gemma","model_variant":"Gemma 2 27B IT","model_version":"Gemma 2","name":"Gemma 2 27B IT","open_source":true,"parameter_size":"27B","provider":"google","url":""},"gemma-2-2b":{"country":"US","id":"gemma-2-2b","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 2B","model_family":"Gemma","model_variant":"Gemma2-2B","model_version":"Gemma 2","name":"Gemma 2 2B","open_source":true,"parameter_size":"2B","provider":"google","url":""},"gemma-2-2b-it":{"country":"US","id":"gemma-2-2b-it","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 2B","model_family":"Gemma","model_variant":"Gemma 2 2B IT","model_version":"Gemma 2","name":"Gemma 2 2B IT","open_source":true,"parameter_size":"2B","provider":"google","url":""},"gemma-2-7b":{"country":"US","id":"gemma-2-7b","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 7B","model_family":"Gemma","model_variant":"Gemma2-7B","model_version":"Gemma 2","name":"Gemma 2 7B","open_source":true,"parameter_size":"7B","provider":"google","url":""},"gemma-2-9b":{"country":"US","id":"gemma-2-9b","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 9B","model_family":"Gemma","model_variant":"Gemma 2","model_version":"Gemma 2","name":"Gemma 2 9B","open_source":true,"parameter_size":"9B","provider":"google","url":""},"gemma-2-9b-instruct":{"country":"US","id":"gemma-2-9b-instruct","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 9B","model_family":"Gemma","model_variant":"Gemma 2 9B Instruct","model_version":"Gemma 2","name":"Gemma 2 9B Instruct","open_source":true,"parameter_size":"9B","provider":"google","url":""},"gemma-2-9b-it":{"country":"US","id":"gemma-2-9b-it","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 9B IT","model_family":"Gemma","model_variant":"Gemma 2 9B IT","model_version":"Gemma 2","name":"Gemma 2 9B IT","open_source":true,"parameter_size":"9B","provider":"google","url":""},"gemma-2-9b-it-simpo":{"country":"US","id":"gemma-2-9b-it-simpo","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 9B","model_family":"Gemma","model_variant":"Gemma 2 9B IT SimPO","model_version":"Gemma 2","name":"Gemma 2 9B IT SimPO","open_source":true,"parameter_size":"9B","provider":"princeton","url":""},"gemma-2b":{"country":"US","id":"gemma-2b","launched":"2024-02-21","modalities":["text"],"model":"gemma-2b","model_family":"Gemma","model_variant":"gemma-2b","model_version":"gemma-2b","name":"gemma-2b","open_source":true,"parameter_size":"2B","provider":"google","url":""},"gemma-2b-it":{"country":"US","id":"gemma-2b-it","launched":"2024-02-21","modalities":["text"],"model":"Gemma 2B","model_family":"Gemma","model_variant":"gemma-2b-it","model_version":"Gemma","name":"gemma-2b-it","open_source":true,"parameter_size":"2B","provider":"google","url":""},"gemma-3":{"country":"US","id":"gemma-3","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 27B","model_family":"Gemma","model_variant":"Gemma3","model_version":"Gemma 3","name":"Gemma 3","open_source":true,"parameter_size":"27B","provider":"google","url":""},"gemma-3-12b":{"country":"US","id":"gemma-3-12b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 12B","model_family":"Gemma","model_variant":"Gemma 3 12B","model_version":"Gemma 3","name":"Gemma 3 12B","open_source":true,"parameter_size":"","provider":"google","url":""},"gemma-3-12b-base":{"country":"US","id":"gemma-3-12b-base","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 12B","model_family":"Gemma","model_variant":"Gemma 3 12B Base","model_version":"Gemma 3","name":"Gemma 3 12B Base","open_source":true,"parameter_size":"12B","provider":"google","url":""},"gemma-3-12b-it":{"country":"US","id":"gemma-3-12b-it","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 12B","model_family":"Gemma","model_variant":"Gemma 3 12B IT","model_version":"Gemma 3","name":"Gemma 3 12B (IT)","open_source":true,"parameter_size":"12B","provider":"google","url":""},"gemma-3-1b":{"country":"US","id":"gemma-3-1b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 1B","model_family":"Gemma","model_variant":"Gemma3-1B","model_version":"Gemma 3","name":"Gemma 3 1B","open_source":true,"parameter_size":"1B","provider":"google","url":""},"gemma-3-27b":{"country":"US","id":"gemma-3-27b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 27B","model_family":"Gemma","model_variant":"Gemma 3 27B","model_version":"Gemma 3","name":"Gemma 3 27B","open_source":true,"parameter_size":"","provider":"google","url":""},"gemma-3-27b-it":{"country":"US","id":"gemma-3-27b-it","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 27B","model_family":"Gemma","model_variant":"Gemma 3 27B IT","model_version":"Gemma 3","name":"Gemma 3 27B (IT)","open_source":true,"parameter_size":"27B","provider":"google","url":""},"gemma-3-27b-vl":{"country":"US","id":"gemma-3-27b-vl","launched":"2025-03-12","modalities":["visual"],"model":"Gemma 3 27B VLM","model_family":"Gemma","model_variant":"Gemma3-VL-27B","model_version":"Gemma 3","name":"Gemma 3 27B VLM","open_source":true,"parameter_size":"27B","provider":"google","url":""},"gemma-3-4b":{"country":"US","id":"gemma-3-4b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 4B","model_family":"Gemma","model_variant":"Gemma 3 4B","model_version":"Gemma 3","name":"Gemma 3 4B","open_source":true,"parameter_size":"","provider":"google","url":""},"gemma-3-4b-base":{"country":"US","id":"gemma-3-4b-base","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 4B","model_family":"Gemma","model_variant":"Gemma 3 4B Base","model_version":"Gemma 3","name":"Gemma 3 4B Base","open_source":true,"parameter_size":"4B","provider":"google","url":""},"gemma-3-4b-it":{"country":"US","id":"gemma-3-4b-it","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 4B","model_family":"Gemma","model_variant":"Gemma 3 4B IT","model_version":"Gemma 3","name":"Gemma 3 4B (IT)","open_source":true,"parameter_size":"4B","provider":"google","url":""},"gemma-3-it-12b":{"country":"US","id":"gemma-3-it-12b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 12B","model_family":"Gemma","model_variant":"Gemma3-12B-IT","model_version":"Gemma 3","name":"Gemma 3 IT 12B","open_source":true,"parameter_size":"12B","provider":"google","url":""},"gemma-3-it-1b":{"country":"US","id":"gemma-3-it-1b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 1B","model_family":"Gemma","model_variant":"Gemma3-1B-IT","model_version":"Gemma 3","name":"Gemma 3 IT 1B","open_source":true,"parameter_size":"1B","provider":"google","url":""},"gemma-3-it-270m":{"country":"US","id":"gemma-3-it-270m","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 270M","model_family":"Gemma","model_variant":"Gemma3-270M-IT","model_version":"Gemma 3","name":"Gemma 3 IT 270M","open_source":true,"parameter_size":"0.27B","provider":"google","url":"https://huggingface.co/google/gemma-3-270m-it"},"gemma-3-it-27b":{"country":"US","id":"gemma-3-it-27b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 27B","model_family":"Gemma","model_variant":"Gemma3-27B","model_version":"Gemma 3","name":"Gemma 3 IT 27B","open_source":true,"parameter_size":"27B","provider":"google","url":""},"gemma-3-it-4b":{"country":"US","id":"gemma-3-it-4b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 4B","model_family":"Gemma","model_variant":"Gemma3-4B-IT","model_version":"Gemma 3","name":"Gemma 3 IT 4B","open_source":true,"parameter_size":"4B","provider":"google","url":""},"gemma-3-pt-1b":{"country":"US","id":"gemma-3-pt-1b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 1B","model_family":"Gemma","model_variant":"Gemma3-1B-PT","model_version":"Gemma 3","name":"Gemma 3 PT 1B","open_source":true,"parameter_size":"1B","provider":"google","url":""},"gemma-3-pt-270m":{"country":"US","id":"gemma-3-pt-270m","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 270M","model_family":"Gemma","model_variant":"Gemma3-270M","model_version":"Gemma 3","name":"Gemma 3 PT 270M","open_source":true,"parameter_size":"0.27B","provider":"google","url":""},"gemma-3-pt-27b":{"country":"US","id":"gemma-3-pt-27b","launched":"2025-03-12","modalities":["text"],"model":"Gemma 3 27B","model_family":"Gemma","model_variant":"Gemma3-27B","model_version":"Gemma 3","name":"Gemma 3 PT 27B","open_source":true,"parameter_size":"27B","provider":"google","url":""},"gemma-3n-e2b":{"country":"US","id":"gemma-3n-e2b","launched":"2025-06-26","modalities":["text"],"model":"Gemma 3n E2B","model_family":"Gemma","model_variant":"Gemma 3n E2B","model_version":"Gemma 3n","name":"Gemma 3n E2B","open_source":true,"parameter_size":"E2B","provider":"google","url":""},"gemma-3n-e2b-instructed-litert-preview":{"country":"US","id":"gemma-3n-e2b-instructed-litert-preview","launched":"","modalities":["text"],"model":"Gemma 3n E2B","model_family":"Gemma","model_variant":"Gemma 3n E2B","model_version":"Gemma 3n","name":"Gemma 3n E2B Instructed LiteRT Preview","open_source":true,"parameter_size":"E2B","provider":"google","url":""},"gemma-3n-e2b-it":{"country":"US","id":"gemma-3n-e2b-it","launched":"","modalities":["text"],"model":"Gemma 3n E2B","model_family":"Gemma","model_variant":"Gemma 3n E2B IT","model_version":"Gemma 3n","name":"Gemma 3n E2B IT","open_source":false,"parameter_size":"","provider":"google","url":""},"gemma-3n-e4b":{"country":"US","id":"gemma-3n-e4b","launched":"2025-06-26","modalities":["text"],"model":"Gemma 3n E4B","model_family":"Gemma","model_variant":"Gemma 3n E4B","model_version":"Gemma 3n","name":"Gemma 3n E4B","open_source":true,"parameter_size":"E4B","provider":"google","url":""},"gemma-3n-e4b-instructed-litert-preview":{"country":"US","id":"gemma-3n-e4b-instructed-litert-preview","launched":"","modalities":["text"],"model":"Gemma 3n E4B","model_family":"Gemma","model_variant":"Gemma 3n E4B Instructed LiteRT Preview","model_version":"Gemma 3n","name":"Gemma 3n E4B Instructed LiteRT Preview","open_source":true,"parameter_size":"E4B","provider":"google","url":""},"gemma-3n-e4b-it":{"country":"US","id":"gemma-3n-e4b-it","launched":"","modalities":["text"],"model":"Gemma 3n E4B","model_family":"Gemma","model_variant":"Gemma 3n E4B IT","model_version":"Gemma 3n","name":"Gemma 3n E4B IT","open_source":false,"parameter_size":"","provider":"google","url":""},"gemma-7b":{"country":"US","id":"gemma-7b","launched":"","modalities":["text"],"model":"gemma-7b","model_family":"gemma-7b","model_variant":"gemma-7b","model_version":"gemma-7b","name":"gemma-7b","open_source":true,"parameter_size":"7B","provider":"google","url":""},"gemma-7b-it":{"country":"US","id":"gemma-7b-it","launched":"2024-02-21","modalities":["text"],"model":"Gemma 7B","model_family":"Gemma","model_variant":"gemma-7b-it","model_version":"Gemma","name":"gemma-7b-it","open_source":true,"parameter_size":"7B","provider":"google","url":""},"gemma2-9b":{"country":"US","id":"gemma2-9b","launched":"2024-06-27","modalities":["text"],"model":"Gemma 2 9B","model_family":"Gemma 2","model_variant":"Gemma 2 9B","model_version":"Gemma 2","name":"Gemma 2 9B","open_source":true,"parameter_size":"9B","provider":"google","url":""},"gigachat-2-max":{"country":"RU","id":"gigachat-2-max","launched":"","modalities":["text"],"model":"GigaChat 2 Max","model_family":"GigaChat","model_variant":"2 Max","model_version":"GigaChat 2","name":"GigaChat 2 Max","open_source":false,"parameter_size":"","provider":"gigachat","url":""},"glaive-swe-v1":{"country":"US","id":"glaive-swe-v1","launched":"","modalities":["text"],"model":"Glaive SWE V1","model_family":"Glaive","model_variant":"SWE V1","model_version":"Glaive SWE","name":"Glaive SWE V1","open_source":false,"parameter_size":"34Be","provider":"glaive","url":""},"glm-4-0520":{"country":"CN","id":"glm-4-0520","launched":"2024-05-20","modalities":["text"],"model":"GLM 4","model_family":"GLM","model_variant":"GLM 4","model_version":"GLM 4","name":"GLM 4 (05-20)","open_source":false,"parameter_size":"","provider":"zai","url":""},"glm-4-32b":{"country":"CN","id":"glm-4-32b","launched":"","modalities":["text"],"model":"GLM 4 32B","model_family":"GLM","model_variant":"GLM 4-32B","model_version":"GLM 4","name":"GLM 4 32B","open_source":true,"parameter_size":"32B","provider":"zai","url":""},"glm-4-9b-chat":{"country":"CN","id":"glm-4-9b-chat","launched":"","modalities":["text"],"model":"GLM 4 9B","model_family":"GLM","model_variant":"GLM 4 9B Chat","model_version":"GLM 4","name":"Zhipu GLM 4-9B-Chat","open_source":true,"parameter_size":"9B","provider":"zai","url":""},"glm-4-flash":{"country":"CN","id":"glm-4-flash","launched":"","modalities":["text"],"model":"GLM 4 Flash","model_family":"GLM","model_variant":"4 Flash","model_version":"GLM 4","name":"GLM 4 Flash","open_source":false,"parameter_size":"","provider":"zai","url":""},"glm-4-plus":{"country":"CN","id":"glm-4-plus","launched":"","modalities":["text"],"model":"GLM 4 Plus","model_family":"GLM","model_variant":"GLM 4 Plus","model_version":"GLM 4","name":"GLM 4 Plus","open_source":false,"parameter_size":"","provider":"zai","url":""},"glm-4-plus-0111":{"country":"CN","id":"glm-4-plus-0111","launched":"2025-01-11","modalities":["text"],"model":"GLM 4 Plus","model_family":"GLM","model_variant":"GLM 4 Plus","model_version":"GLM 4","name":"GLM 4 Plus (01-11)","open_source":false,"parameter_size":"","provider":"zai","url":""},"glm-4.1v":{"country":"CN","id":"glm-4.1v","launched":"2025-07-02","modalities":["visual"],"model":"GLM 4.1V","model_family":"GLM","model_variant":"GLM-4.1V","model_version":"GLM 4.1V","name":"GLM-4.1V","open_source":true,"parameter_size":"355B","provider":"zai","url":""},"glm-4.1v-9b":{"country":"CN","id":"glm-4.1v-9b","launched":"2025-07-02","modalities":["visual"],"model":"GLM 4.1V 9B","model_family":"GLM","model_variant":"GLM-4.1V-9B","model_version":"GLM 4.1V","name":"GLM-4.1V-9B","open_source":true,"parameter_size":"9B","provider":"zai","url":""},"glm-4.1v-9b-thinking":{"country":"CN","id":"glm-4.1v-9b-thinking","launched":"2025-07-02","modalities":["visual"],"model":"GLM 4.1V 9B Thinking","model_family":"GLM","model_variant":"GLM-4.1V-9B-Thinking","model_version":"GLM 4.1V","name":"GLM-4.1V-9B-Thinking","open_source":true,"parameter_size":"9B","provider":"zai","url":""},"glm-4.5":{"country":"CN","id":"glm-4.5","launched":"2025-07-28","modalities":["text"],"model":"GLM 4.5","model_family":"GLM","model_variant":"GLM 4.5","model_version":"GLM 4.5","name":"GLM 4.5","open_source":true,"parameter_size":"355B","provider":"zai","url":""},"glm-4.5-358b":{"country":"CN","id":"glm-4.5-358b","launched":"2025-07-28","modalities":["text"],"model":"GLM 4.5","model_family":"GLM","model_variant":"GLM 4.5 358B","model_version":"GLM 4.5","name":"GLM-4.5-358B","open_source":true,"parameter_size":"358B","provider":"zai","url":""},"glm-4.5-358b-mini-swe-agent":{"country":"CN","id":"glm-4.5-358b-mini-swe-agent","launched":"2025-07-28","modalities":["text"],"model":"GLM 4.5","model_family":"GLM","model_variant":"GLM 4.5 358B mini-SWE-Agent","model_version":"GLM 4.5","name":"GLM-4.5-358B (mini-SWE-Agent)","open_source":true,"parameter_size":"358B","provider":"zai","url":""},"glm-4.5-358b-swe-agent":{"country":"CN","id":"glm-4.5-358b-swe-agent","launched":"2025-07-28","modalities":["text"],"model":"GLM 4.5","model_family":"GLM","model_variant":"GLM 4.5 358B SWE-Agent","model_version":"GLM 4.5","name":"GLM-4.5-358B (SWE-Agent)","open_source":true,"parameter_size":"358B","provider":"zai","url":""},"glm-4.5-air":{"active_parameters":"12B","country":"CN","id":"glm-4.5-air","launched":"2025-08-11","modalities":["text"],"model":"GLM 4.5 Air","model_family":"GLM","model_variant":"GLM-4.5-Air","model_version":"GLM 4.5","name":"GLM-4.5-Air","open_source":true,"parameter_size":"110B","provider":"zai","url":""},"glm-4.5-base":{"country":"CN","id":"glm-4.5-base","launched":"2025-07-28","modalities":["text"],"model":"GLM 4.5","model_family":"GLM","model_variant":"GLM-4.5 Base","model_version":"GLM 4.5","name":"GLM-4.5 Base","open_source":true,"parameter_size":"355B","provider":"zai","url":""},"glm-4.5v":{"country":"CN","id":"glm-4.5v","launched":"2025-07-28","modalities":["text","visual"],"model":"GLM 4.5V","model_family":"GLM","model_variant":"GLM 4.5","model_version":"GLM 4.5V","name":"GLM-4.5V","open_source":true,"parameter_size":"160B","provider":"zai","url":""},"glm-4.6":{"country":"CN","id":"glm-4.6","launched":"2025-09-29","modalities":["text"],"model":"GLM 4.6","model_family":"GLM","model_variant":"GLM 4.6","model_version":"GLM 4.6","name":"GLM 4.6","open_source":true,"parameter_size":"357B","provider":"zai","url":""},"glm-4.6v":{"active_parameters":"12B","country":"CN","id":"glm-4.6v","launched":"2025-12-08","modalities":["text","visual"],"model":"GLM 4.6V","model_family":"GLM","model_variant":"GLM 4.6V","model_version":"GLM 4.6V","name":"GLM-4.6V","open_source":true,"parameter_size":"106B","provider":"zai","url":""},"glm-4.6v-flash":{"country":"CN","id":"glm-4.6v-flash","launched":"2025-12-08","modalities":["text","visual"],"model":"GLM 4.6V Flash","model_family":"GLM","model_variant":"GLM 4.6V Flash","model_version":"GLM 4.6V","name":"GLM-4.6V-Flash","open_source":true,"parameter_size":"9B","provider":"zai","url":""},"glm-4.7":{"country":"CN","id":"glm-4.7","launched":"2025-12-22","modalities":["text"],"model":"GLM 4.7","model_family":"GLM","model_variant":"GLM 4.7","model_version":"GLM 4.7","name":"GLM 4.7","open_source":true,"parameter_size":"","provider":"zai","url":""},"glm-4.7-flash":{"active_parameters":"3B","country":"CN","id":"glm-4.7-flash","launched":"2026-01-19","modalities":["text"],"model":"GLM 4.7 Flash","model_family":"GLM","model_variant":"GLM 4.7 Flash","model_version":"GLM 4.7","name":"GLM-4.7-Flash","open_source":true,"parameter_size":"30B","provider":"zai","url":"https://huggingface.co/zai-org/GLM-4.7-Flash"},"glm-4.7-thinking":{"country":"CN","id":"glm-4.7-thinking","launched":"2025-12-22","modalities":["text"],"model":"GLM 4.7","model_family":"GLM","model_variant":"GLM 4.7 Thinking","model_version":"GLM 4.7","name":"GLM 4.7 Thinking","open_source":true,"parameter_size":"","provider":"zai","url":""},"glm-5":{"active_parameters":"40B","country":"CN","id":"glm-5","launched":"2026-02-11","modalities":["text"],"model":"GLM-5","model_family":"GLM","model_variant":"GLM-5","model_version":"GLM 5","name":"GLM-5","open_source":true,"parameter_size":"744B","provider":"zai","url":""},"glm-image":{"country":"CN","id":"glm-image","launched":"2026-01-14","modalities":["text","visual"],"model":"GLM-Image","model_family":"GLM","model_variant":"GLM-Image","model_version":"GLM-Image","name":"GLM-Image","open_source":true,"parameter_size":"16B","provider":"zai","url":"https://huggingface.co/zai-org/GLM-Image"},"glm-z1":{"country":"CN","id":"glm-z1","launched":"2025-04-14","modalities":["text"],"model":"GLM Z1","model_family":"GLM","model_variant":"Z1","model_version":"GLM Z1","name":"GLM-Z1","open_source":true,"parameter_size":"9B","provider":"zai","url":""},"glm4-9b":{"country":"CN","id":"glm4-9b","launched":"","modalities":["text"],"model":"GLM 4 9B","model_family":"GLM","model_variant":"GLM 4 9B","model_version":"GLM 4","name":"GLM 4 9B","open_source":true,"parameter_size":"9B","provider":"zai","url":""},"gopher-280b":{"country":"US","id":"gopher-280b","launched":"2021-12-08","modalities":["text"],"model":"Gopher 280B","model_family":"Gopher","model_variant":"280B","model_version":"Gopher","name":"Gopher 280B","open_source":false,"parameter_size":"280B","provider":"google","url":""},"got-ocr":{"country":"CN","id":"got-ocr","launched":"","modalities":["visual"],"model":"GOT-OCR","model_family":"GOT-OCR","model_variant":"1.0","model_version":"GOT-OCR","name":"GOT-OCR","open_source":true,"parameter_size":"","provider":"stepfun","url":""},"got-ocr2.0":{"country":"CN","id":"got-ocr2.0","launched":"","modalities":["visual"],"model":"GOT-OCR2.0","model_family":"GOT-OCR","model_variant":"2.0","model_version":"GOT-OCR","name":"GOT-OCR2.0","open_source":true,"parameter_size":"0.7B","provider":"stepfun","url":""},"gpt-03-cua":{"country":"US","id":"gpt-03-cua","launched":"2025-01-31","modalities":["text"],"model":"o3 Operator","model_family":"o3","model_variant":"o3 Operator","model_version":"o3","name":"o3 Operator","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-2":{"country":"US","id":"gpt-2","launched":"2019-02-14","modalities":["text"],"model":"GPT-2","model_family":"GPT","model_variant":"GPT-2","model_version":"GPT-2","name":"GPT-2","open_source":true,"parameter_size":"1.5B","provider":"openai","url":""},"gpt-3":{"country":"US","id":"gpt-3","launched":"2020-05-28","modalities":["text"],"model":"GPT-3","model_family":"GPT","model_variant":"GPT-3","model_version":"GPT-3","name":"GPT-3","open_source":false,"parameter_size":"175B","provider":"openai","url":""},"gpt-3.5":{"country":"US","id":"gpt-3.5","launched":"2022-11-30","modalities":["text"],"model":"GPT 3.5","model_family":"GPT","model_variant":"GPT 3.5","model_version":"GPT-3","name":"GPT 3.5","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-3.5-may-2023":{"country":"US","id":"gpt-3.5-may-2023","launched":"","modalities":["text"],"model":"GPT-3.5","model_family":"GPT","model_variant":"GPT-3.5 (May 2023)","model_version":"GPT-3.5","name":"GPT-3.5 (May 2023)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-3.5-turbo":{"country":"US","id":"gpt-3.5-turbo","launched":"2023-03-01","modalities":["text"],"model":"GPT-3.5 Turbo","model_family":"GPT","model_variant":"GPT-3.5","model_version":"GPT-3.5","name":"GPT-3.5-turbo","open_source":false,"parameter_size":"175Be","provider":"openai","url":""},"gpt-3.5-turbo-0125":{"country":"US","id":"gpt-3.5-turbo-0125","launched":"","modalities":["text"],"model":"GPT-3.5 Turbo","model_family":"GPT","model_variant":"GPT 3.5 Turbo 0125","model_version":"GPT-3.5","name":"GPT-3.5-Turbo-0125","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-3.5-turbo-0301":{"country":"US","id":"gpt-3.5-turbo-0301","launched":"2023-03-01","modalities":["text"],"model":"GPT-3.5 Turbo","model_family":"GPT","model_variant":"GPT-3.5 Turbo 0301","model_version":"GPT-3.5","name":"GPT-3.5 Turbo (03-01)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-3.5-turbo-0613":{"country":"US","id":"gpt-3.5-turbo-0613","launched":"2023-06-13","modalities":["text"],"model":"GPT-3.5 Turbo","model_family":"GPT","model_variant":"GPT-3.5 Turbo 0613","model_version":"GPT-3.5","name":"GPT-3.5-Turbo-0613","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-3.5-turbo-0613-cot":{"country":"US","id":"gpt-3.5-turbo-0613-cot","launched":"2023-06-13","modalities":["text"],"model":"GPT-3.5 Turbo CoT","model_family":"GPT","model_variant":"GPT-3.5 Turbo 0613 CoT","model_version":"GPT-3.5","name":"GPT-3.5 Turbo 0613 CoT","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-3.5-turbo-1106":{"country":"US","id":"gpt-3.5-turbo-1106","launched":"2023-11-06","modalities":["text"],"model":"GPT-3.5 Turbo","model_family":"GPT","model_variant":"GPT-3.5 Turbo 1106","model_version":"GPT-3.5","name":"GPT-3.5 Turbo (1106)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-3.5-turbo-nov-2023":{"country":"US","id":"gpt-3.5-turbo-nov-2023","launched":"","modalities":["text"],"model":"GPT-3.5 Turbo","model_family":"GPT","model_variant":"GPT-3.5-Turbo (Nov 2023)","model_version":"GPT-3.5","name":"GPT-3.5-Turbo (Nov 2023)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4":{"country":"US","id":"gpt-4","launched":"2023-03-14","modalities":["text"],"model":"GPT-4","model_family":"GPT","model_variant":"GPT-4","model_version":"GPT-4","name":"GPT-4","open_source":false,"parameter_size":"1760Be","provider":"openai","url":""},"gpt-4-0125-preview":{"country":"US","id":"gpt-4-0125-preview","launched":"2024-01-25","modalities":["text"],"model":"GPT-4","model_family":"GPT","model_variant":"4 0125 Preview","model_version":"GPT-4","name":"GPT-4 0125 Preview","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4-0314":{"country":"US","id":"gpt-4-0314","launched":"2023-03-14","modalities":["text"],"model":"GPT-4","model_family":"GPT","model_variant":"GPT-4 0314","model_version":"GPT-4","name":"GPT-4 (03-14)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4-0613":{"country":"US","id":"gpt-4-0613","launched":"","modalities":["text"],"model":"GPT-4","model_family":"GPT","model_variant":"GPT 4 0613","model_version":"GPT-4","name":"GPT 4 0613","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4-0613-cot":{"country":"US","id":"gpt-4-0613-cot","launched":"","modalities":["text"],"model":"GPT 4 0613 CoT","model_family":"GPT","model_variant":"GPT 4 0613 CoT","model_version":"GPT-3","name":"GPT-4 0613 CoT","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4-1106-preview":{"country":"US","id":"gpt-4-1106-preview","launched":"2024-05-13","modalities":["text"],"model":"GPT-4 Turbo","model_family":"GPT","model_variant":"GPT-4 Turbo","model_version":"GPT-4","name":"GPT-4 Turbo (11-06)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4-turbo":{"country":"US","id":"gpt-4-turbo","launched":"2023-11-06","modalities":["text"],"model":"GPT-4 Turbo","model_family":"GPT","model_variant":"GPT-4","model_version":"GPT-4","name":"GPT-4 Turbo","open_source":false,"parameter_size":"1760Be","provider":"openai","url":""},"gpt-4-turbo-2024-04-09":{"country":"US","id":"gpt-4-turbo-2024-04-09","launched":"2024-04-09","modalities":["text"],"model":"GPT-4 Turbo","model_family":"GPT","model_variant":"GPT-4 Turbo 20240409","model_version":"GPT-4","name":"GPT-4 Turbo (2024-04-09)","open_source":false,"parameter_size":"1760Be","provider":"openai","url":""},"gpt-4-turbo-2024-04-09-cot":{"country":"US","id":"gpt-4-turbo-2024-04-09-cot","launched":"2024-04-09","modalities":["text"],"model":"GPT-4 Turbo CoT","model_family":"GPT","model_variant":"GPT-4 Turbo 20240409 CoT","model_version":"GPT-4","name":"GPT-4 Turbo (2024-04-09) CoT","open_source":false,"parameter_size":"1760Be","provider":"openai","url":""},"gpt-4.0613":{"country":"US","id":"gpt-4.0613","launched":"","modalities":["text"],"model":"GPT-4","model_family":"GPT","model_variant":"GPT 4.0613","model_version":"GPT-4","name":"GPT 4.0613","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4.1":{"country":"US","id":"gpt-4.1","launched":"2025-04-14","modalities":["text","visual"],"model":"GPT-4.1","model_family":"GPT","model_variant":"GPT-4","model_version":"GPT-4.1","name":"GPT-4.1","open_source":false,"parameter_size":"1760Be","provider":"openai","url":""},"gpt-4.1-2025-04-14":{"country":"US","id":"gpt-4.1-2025-04-14","launched":"2025-04-14","modalities":["text","visual"],"model":"GPT-4.1","model_family":"GPT","model_variant":"GPT-4.1 20250414","model_version":"GPT-4.1","name":"GPT-4.1 (2025-04-14)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4.1-mini":{"country":"US","id":"gpt-4.1-mini","launched":"2025-04-14","modalities":["text"],"model":"GPT-4.1 Mini","model_family":"GPT","model_variant":"GPT-4.1 mini","model_version":"GPT-4.1","name":"GPT-4.1 mini","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4.1-mini-2025-04-14":{"country":"US","id":"gpt-4.1-mini-2025-04-14","launched":"2025-04-14","modalities":["text"],"model":"GPT-4.1 Mini","model_family":"GPT","model_variant":"GPT 4.1 Mini 2025 04 14","model_version":"GPT-4.1","name":"GPT-4.1-Mini-2025-04-14","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4.1-nano":{"country":"US","id":"gpt-4.1-nano","launched":"2025-04-14","modalities":["text"],"model":"GPT-4.1 Nano","model_family":"GPT","model_variant":"GPT-4.1 nano","model_version":"GPT-4.1","name":"GPT-4.1 nano","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4.1-nano-2025-04-14":{"country":"US","id":"gpt-4.1-nano-2025-04-14","launched":"2025-04-14","modalities":["text"],"model":"GPT-4.1 Nano","model_family":"GPT","model_variant":"GPT 4.1 Nano 2025 04 14","model_version":"GPT-4.1","name":"GPT-4.1-Nano-2025-04-14","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4.5":{"country":"US","id":"gpt-4.5","launched":"2025-02-27","modalities":["text"],"model":"GPT-4.5","model_family":"GPT","model_variant":"GPT-4","model_version":"GPT-4.5","name":"GPT-4.5","open_source":false,"parameter_size":"500Be","provider":"openai","url":""},"gpt-4.5-preview":{"country":"US","id":"gpt-4.5-preview","launched":"2025-02-27","modalities":["text"],"model":"GPT-4.5","model_family":"GPT","model_variant":"GPT-4.5 Preview","model_version":"GPT-4.5","name":"GPT-4.5 Preview","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4.5-preview-2025-02-27":{"country":"US","id":"gpt-4.5-preview-2025-02-27","launched":"2025-02-27","modalities":["text"],"model":"GPT-4.5","model_family":"GPT","model_variant":"GPT-4.5 Preview","model_version":"GPT-4.5","name":"GPT-4.5 Preview (2025-02-27)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4o":{"country":"US","id":"gpt-4o","launched":"2024-05-13","modalities":["text","visual"],"model":"GPT-4o","model_family":"GPT","model_variant":"Gpt-4o","model_version":"GPT-4o","name":"GPT-4o","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-4o-0327":{"country":"US","id":"gpt-4o-0327","launched":"2024-05-13","modalities":["text","visual"],"model":"GPT-4o","model_family":"GPT","model_variant":"GPT-4o 0327","model_version":"GPT-4o","name":"GPT-4o (0327)","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-4o-0513":{"country":"US","id":"gpt-4o-0513","launched":"2024-05-13","modalities":["text","visual"],"model":"GPT-4o","model_family":"GPT","model_variant":"GPT-4o 0513","model_version":"GPT-4o","name":"GPT-4o (0513)","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-4o-2024-05-13":{"country":"US","id":"gpt-4o-2024-05-13","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"GPT 4o 2024 05 13","model_version":"GPT-4o","name":"GPT-4o-2024-05-13","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4o-2024-08-06":{"country":"US","id":"gpt-4o-2024-08-06","launched":"2024-05-13","modalities":["text","visual"],"model":"GPT-4o","model_family":"GPT","model_variant":"GPT-4o 20240806","model_version":"GPT-4o","name":"GPT-4o (2024-08-06)","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-4o-2024-11-20":{"country":"US","id":"gpt-4o-2024-11-20","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"GPT 4o 2024 11 20","model_version":"GPT-4o","name":"GPT-4o-2024-11-20","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4o-2024-11-24":{"country":"US","id":"gpt-4o-2024-11-24","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o","model_family":"GPT","model_variant":"GPT 4o 2024 11 24","model_version":"GPT-4o","name":"GPT-4o-2024-11-24","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4o-audio":{"country":"US","id":"gpt-4o-audio","launched":"2024-05-13","modalities":["audio","text"],"model":"GPT-4o Audio","model_family":"GPT","model_variant":"GPT-4o Audio","model_version":"GPT-4o","name":"GPT-4o Audio","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-4o-cot":{"country":"US","id":"gpt-4o-cot","launched":"2024-05-13","modalities":["text","visual"],"model":"GPT-4o CoT","model_family":"GPT","model_variant":"GPT-4o CoT","model_version":"GPT-4o","name":"GPT-4o CoT","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-4o-mini":{"country":"US","id":"gpt-4o-mini","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o Mini","model_family":"GPT","model_variant":"Gpt-4o mini","model_version":"GPT-4o","name":"GPT-4o mini","open_source":false,"parameter_size":"8Be","provider":"openai","url":""},"gpt-4o-mini-2024-07-18":{"country":"US","id":"gpt-4o-mini-2024-07-18","launched":"2024-05-13","modalities":["text"],"model":"GPT-4o Mini","model_family":"GPT","model_variant":"GPT-4o mini 20240718","model_version":"GPT-4o","name":"GPT-4o mini (2024-07-18)","open_source":false,"parameter_size":"8Be","provider":"openai","url":""},"gpt-4o-r2e-gym":{"country":"US","id":"gpt-4o-r2e-gym","launched":"2024-05-13","modalities":["text","visual"],"model":"GPT-4o","model_family":"GPT","model_variant":"Gpt-4o R2E-Gym","model_version":"GPT-4o","name":"GPT-4o (R2E-Gym)","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-4v":{"country":"US","id":"gpt-4v","launched":"","modalities":["visual"],"model":"GPT-4V","model_family":"GPT","model_variant":"GPT-4V","model_version":"GPT-4V","name":"GPT-4V(ision)","open_source":false,"parameter_size":"1760Be","provider":"openai","url":""},"gpt-5":{"country":"US","id":"gpt-5","launched":"2025-08-07","modalities":["text","visual"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5","model_version":"GPT-5","name":"GPT-5","open_source":false,"parameter_size":"10000Be","provider":"openai","url":""},"gpt-5-2025-08-07":{"country":"US","id":"gpt-5-2025-08-07","launched":"2025-08-07","modalities":["text","visual"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 20250807","model_version":"GPT-5","name":"GPT-5 (2025-08-07)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-chat":{"country":"US","id":"gpt-5-chat","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Chat","model_version":"GPT-5","name":"GPT-5 Chat","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-codex":{"country":"US","id":"gpt-5-codex","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Codex","model_family":"GPT","model_variant":"GPT-5 Codex","model_version":"GPT-5","name":"GPT-5 Codex","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-codex-high":{"country":"US","id":"gpt-5-codex-high","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Codex","model_family":"GPT","model_variant":"GPT-5 Codex High","model_version":"GPT-5","name":"GPT-5 Codex High","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-codex-low":{"country":"US","id":"gpt-5-codex-low","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Codex","model_family":"GPT","model_variant":"GPT-5 Codex Low","model_version":"GPT-5","name":"GPT-5 Codex Low","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-codex-medium":{"country":"US","id":"gpt-5-codex-medium","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Codex","model_family":"GPT","model_variant":"GPT-5 Codex Medium","model_version":"GPT-5","name":"GPT-5 Codex Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-high":{"country":"US","id":"gpt-5-high","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 High","model_version":"GPT-5","name":"GPT-5 High","open_source":false,"parameter_size":"10000Be","provider":"openai","url":""},"gpt-5-high-code":{"country":"US","id":"gpt-5-high-code","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 High Code","model_version":"GPT-5","name":"GPT-5 (High, Code)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-high-code-web":{"country":"US","id":"gpt-5-high-code-web","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 High Code \u0026 Web","model_version":"GPT-5","name":"GPT-5 (High, Code \u0026 Web)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-low":{"country":"US","id":"gpt-5-low","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Low","model_version":"GPT-5","name":"GPT-5 Low","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-medium":{"country":"US","id":"gpt-5-medium","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Medium","model_version":"GPT-5","name":"GPT-5 Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-mini":{"country":"US","id":"gpt-5-mini","launched":"2025-08-07","modalities":["visual"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5","model_version":"GPT-5","name":"GPT-5 mini","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-5-mini-2025-08":{"country":"US","id":"gpt-5-mini-2025-08","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Mini 2025-08","model_version":"GPT-5","name":"GPT-5 Mini (2025-08)","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-5-mini-high":{"country":"US","id":"gpt-5-mini-high","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Mini High","model_version":"GPT-5","name":"GPT-5 Mini High","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-mini-low":{"country":"US","id":"gpt-5-mini-low","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Mini Low","model_version":"GPT-5","name":"GPT-5 Mini Low","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-mini-medium":{"country":"US","id":"gpt-5-mini-medium","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Mini Medium","model_version":"GPT-5","name":"GPT-5 Mini Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-mini-minimal":{"country":"US","id":"gpt-5-mini-minimal","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Mini Minimal","model_version":"GPT-5","name":"GPT-5 Mini Minimal","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-minimal":{"country":"US","id":"gpt-5-minimal","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 minimal","model_version":"GPT-5","name":"GPT-5 (minimal)","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-5-minimal-think":{"country":"US","id":"gpt-5-minimal-think","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 minimal Think","model_version":"GPT-5","name":"GPT-5 (minimal) Think","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"gpt-5-nano":{"country":"US","id":"gpt-5-nano","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Nano","model_family":"GPT","model_variant":"GPT-5","model_version":"GPT-5","name":"GPT-5 Nano","open_source":false,"parameter_size":"100Be","provider":"openai","url":""},"gpt-5-nano-high":{"country":"US","id":"gpt-5-nano-high","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Nano","model_family":"GPT","model_variant":"GPT-5 Nano High","model_version":"GPT-5","name":"GPT-5 Nano High","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-nano-low":{"country":"US","id":"gpt-5-nano-low","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Nano","model_family":"GPT","model_variant":"GPT-5 Nano Low","model_version":"GPT-5","name":"GPT-5 Nano Low","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-nano-medium":{"country":"US","id":"gpt-5-nano-medium","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Nano","model_family":"GPT","model_variant":"GPT-5 Nano Medium","model_version":"GPT-5","name":"GPT-5 Nano Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-nano-minimal":{"country":"US","id":"gpt-5-nano-minimal","launched":"2025-08-07","modalities":["text"],"model":"GPT-5 Nano","model_family":"GPT","model_variant":"GPT-5 Nano Minimal","model_version":"GPT-5","name":"GPT-5 Nano Minimal","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-pro":{"country":"US","id":"gpt-5-pro","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5","model_version":"GPT-5","name":"GPT-5 pro","open_source":false,"parameter_size":"5000Be","provider":"openai","url":""},"gpt-5-r2e-gym":{"country":"US","id":"gpt-5-r2e-gym","launched":"2025-08-07","modalities":["text","visual"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 R2E-Gym","model_version":"GPT-5","name":"GPT-5 (R2E-Gym)","open_source":false,"parameter_size":"10000Be","provider":"openai","url":""},"gpt-5-thinking":{"country":"US","id":"gpt-5-thinking","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT-5 Thinking","model_version":"GPT-5","name":"GPT-5 Thinking","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5-thinking-high":{"country":"US","id":"gpt-5-thinking-high","launched":"2025-08-07","modalities":["text"],"model":"GPT-5","model_family":"GPT","model_variant":"GPT5 Thinking High","model_version":"GPT-5","name":"GPT-5 Thinking High","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.1":{"country":"US","id":"gpt-5.1","launched":"2025-11-18","modalities":["text","visual"],"model":"GPT-5.1","model_family":"GPT","model_variant":"GPT-5.1","model_version":"GPT-5.1","name":"GPT-5.1","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.1-codex":{"country":"US","id":"gpt-5.1-codex","launched":"2025-11-18","modalities":["text"],"model":"GPT-5.1 Codex","model_family":"GPT","model_variant":"GPT-5.1 Codex","model_version":"GPT-5.1 Codex","name":"GPT-5.1 Codex","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.1-codex-max":{"country":"US","id":"gpt-5.1-codex-max","launched":"2025-11-19","modalities":["text"],"model":"GPT-5.1 Codex-Max","model_family":"GPT","model_variant":"GPT-5.1 Codex-Max","model_version":"GPT-5.1 Codex","name":"GPT-5.1 Codex-Max","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.1-codex-mini-high":{"country":"US","id":"gpt-5.1-codex-mini-high","launched":"","modalities":["text"],"model":"GPT-5.1 Codex Mini","model_family":"GPT","model_variant":"GPT-5.1 Codex Mini High","model_version":"GPT-5.1 Codex","name":"GPT-5.1 Codex Mini (High)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.1-high":{"country":"US","id":"gpt-5.1-high","launched":"2025-11-18","modalities":["text"],"model":"GPT-5.1","model_family":"GPT","model_variant":"GPT-5.1 High","model_version":"GPT-5.1","name":"GPT-5.1 High","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.1-medium":{"country":"US","id":"gpt-5.1-medium","launched":"2025-11-18","modalities":["text"],"model":"GPT-5.1","model_family":"GPT","model_variant":"GPT-5.1 Medium","model_version":"GPT-5.1","name":"GPT-5.1 Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.1-thinking":{"country":"US","id":"gpt-5.1-thinking","launched":"2025-11-18","modalities":["text"],"model":"GPT-5.1","model_family":"GPT","model_variant":"GPT-5.1 Thinking","model_version":"GPT-5.1","name":"GPT-5.1 Thinking","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2":{"country":"US","id":"gpt-5.2","launched":"2025-12-11","modalities":["text","visual"],"model":"GPT-5.2","model_family":"GPT","model_variant":"GPT-5.2","model_version":"GPT-5.2","name":"GPT-5.2","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2-codex":{"country":"US","id":"gpt-5.2-codex","launched":"2025-12-18","modalities":["text"],"model":"GPT-5.2 Codex","model_family":"GPT","model_variant":"GPT-5.2 Codex","model_version":"GPT-5.2 Codex","name":"GPT-5.2 Codex","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2-extra-high":{"country":"US","id":"gpt-5.2-extra-high","launched":"2025-12-11","modalities":["text"],"model":"GPT-5.2","model_family":"GPT","model_variant":"GPT-5.2 Extra High","model_version":"GPT-5.2","name":"GPT-5.2 Extra High","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2-high":{"country":"US","id":"gpt-5.2-high","launched":"2025-12-11","modalities":["text"],"model":"GPT-5.2","model_family":"GPT","model_variant":"GPT-5.2 High","model_version":"GPT-5.2","name":"GPT-5.2 High","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2-instant":{"country":"US","id":"gpt-5.2-instant","launched":"","modalities":["text"],"model":"GPT-5.2","model_family":"GPT","model_variant":"GPT-5.2 Instant","model_version":"GPT-5.2","name":"GPT-5.2 Instant","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2-medium":{"country":"US","id":"gpt-5.2-medium","launched":"2025-12-11","modalities":["text"],"model":"GPT-5.2","model_family":"GPT","model_variant":"GPT-5.2 Medium","model_version":"GPT-5.2","name":"GPT-5.2 Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2-thinking":{"country":"US","id":"gpt-5.2-thinking","launched":"2025-12-11","modalities":["text"],"model":"GPT-5.2","model_family":"GPT","model_variant":"GPT-5.2 Thinking","model_version":"GPT-5.2","name":"GPT-5.2 Thinking","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.2-thinking-xhigh":{"country":"US","id":"gpt-5.2-thinking-xhigh","launched":"2025-12-11","modalities":["text"],"model":"GPT-5.2","model_family":"GPT","model_variant":"GPT-5.2 Thinking XHigh","model_version":"GPT-5.2","name":"GPT-5.2 Thinking XHigh","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.3-codex":{"country":"US","id":"gpt-5.3-codex","launched":"2026-02-05","modalities":["text"],"model":"GPT-5.3 Codex","model_family":"GPT","model_variant":"GPT-5.3 Codex","model_version":"GPT-5.3 Codex","name":"GPT-5.3 Codex","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-5.3-instant":{"country":"US","id":"gpt-5.3-instant","launched":"","modalities":["text"],"model":"GPT-5.3","model_family":"GPT","model_variant":"GPT-5.3 Instant","model_version":"GPT-5.3","name":"GPT-5.3 Instant","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-image-1-high":{"country":"US","id":"gpt-image-1-high","launched":"2025-03-25","modalities":["visual"],"model":"GPT Image 1","model_family":"GPT Image","model_variant":"High","model_version":"GPT Image 1","name":"GPT Image 1 (High)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-j-6b":{"country":"US","id":"gpt-j-6b","launched":"","modalities":["text"],"model":"GPT-J 6B","model_family":"GPT","model_variant":"GPT-J-6B","model_version":"GPT-J","name":"GPT-J-6B","open_source":false,"parameter_size":"6B","provider":"openai","url":""},"gpt-neo-2.7b":{"country":"US","id":"gpt-neo-2.7b","launched":"","modalities":["text"],"model":"GPT Neo 2.7B","model_family":"GPT","model_variant":"GPT-Neo-2.7B","model_version":"GPT-Neo","name":"GPT-Neo-2.7B","open_source":false,"parameter_size":"2.7B","provider":"openai","url":""},"gpt-o3":{"country":"US","id":"gpt-o3","launched":"2025-06-10","modalities":["text"],"model":"o3","model_family":"o3","model_variant":"O3","model_version":"o3","name":"GPT-O3","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-o3-cua":{"country":"US","id":"gpt-o3-cua","launched":"","modalities":["text","visual"],"model":"o3","model_family":"o3","model_variant":"o3 Operator (CUA)","model_version":"o3","name":"o3 Operator (CUA)","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt-oss-120b":{"country":"US","id":"gpt-oss-120b","launched":"2025-08-05","modalities":["text"],"model":"GPT-OSS 120B","model_family":"GPT","model_variant":"GPT-OSS-120B","model_version":"GPT-OSS","name":"GPT-OSS 120B","open_source":true,"parameter_size":"120B","provider":"openai","url":""},"gpt-oss-120b-2025-08-05":{"country":"US","id":"gpt-oss-120b-2025-08-05","launched":"2025-08-05","modalities":["text"],"model":"GPT-OSS 120B","model_family":"GPT","model_variant":"GPT-OSS 120B","model_version":"GPT-OSS","name":"GPT-OSS 120B (2025-08-05)","open_source":true,"parameter_size":"120B","provider":"openai","url":""},"gpt-oss-120b-high":{"country":"US","id":"gpt-oss-120b-high","launched":"2025-08-05","modalities":["text"],"model":"GPT-OSS 120B","model_family":"GPT","model_variant":"GPT-OSS 120B High","model_version":"GPT-OSS","name":"GPT-OSS 120B (High)","open_source":true,"parameter_size":"120B","provider":"openai","url":""},"gpt-oss-120b-medium":{"country":"","id":"gpt-oss-120b-medium","launched":"","modalities":["text"],"model":"GPT-OSS 120B","model_family":"GPT-OSS","model_variant":"120B Medium","model_version":"GPT-OSS","name":"GPT-OSS 120B Medium","open_source":true,"parameter_size":"120B","provider":"gpt-oss","url":""},"gpt-oss-20b":{"country":"US","id":"gpt-oss-20b","launched":"2025-08-04","modalities":["text"],"model":"GPT-OSS 20B","model_family":"GPT","model_variant":"GPT-OSS-20B","model_version":"GPT-OSS","name":"GPT-OSS 20B","open_source":true,"parameter_size":"20B","provider":"openai","url":""},"gpt-oss-20b-2025-08-05":{"country":"US","id":"gpt-oss-20b-2025-08-05","launched":"2025-08-04","modalities":["text"],"model":"GPT-OSS 20B","model_family":"GPT","model_variant":"GPT-OSS 20B","model_version":"GPT-OSS","name":"GPT-OSS 20B (2025-08-05)","open_source":true,"parameter_size":"20B","provider":"openai","url":""},"gpt-oss-20b-a4b":{"country":"US","id":"gpt-oss-20b-a4b","launched":"2025-08-04","modalities":["text"],"model":"GPT-OSS 20B","model_family":"GPT","model_variant":"GPT-OSS-20B-A4B","model_version":"GPT-OSS","name":"GPT-OSS-20B-A4B","open_source":true,"parameter_size":"20B","provider":"openai","url":""},"gpt-oss-20b-high":{"country":"US","id":"gpt-oss-20b-high","launched":"2025-08-04","modalities":["text"],"model":"GPT-OSS 20B","model_family":"GPT","model_variant":"GPT-OSS-20B High","model_version":"GPT-OSS","name":"GPT-OSS-20B (High)","open_source":true,"parameter_size":"20B","provider":"openai","url":""},"gpt-oss-20b-medium":{"country":"US","id":"gpt-oss-20b-medium","launched":"2025-08-04","modalities":["text"],"model":"GPT-OSS 20B","model_family":"GPT","model_variant":"20B-Medium","model_version":"GPT-OSS","name":"GPT-OSS-20B-Medium","open_source":true,"parameter_size":"20B","provider":"openai","url":""},"gpt-oss-20b-moe":{"country":"US","id":"gpt-oss-20b-moe","launched":"2025-08-04","modalities":["text"],"model":"GPT-OSS 20B","model_family":"GPT","model_variant":"GPT-OSS-20B MoE","model_version":"GPT-OSS","name":"GPT-OSS 20B (MoE)","open_source":true,"parameter_size":"20B","provider":"openai","url":""},"gpt-super-105-turbo":{"country":"US","id":"gpt-super-105-turbo","launched":"","modalities":["text"],"model":"GPT Super 105 Turbo","model_family":"GPT","model_variant":"Super 105 Turbo","model_version":"GPT Super 105","name":"GPT Super 105 Turbo","open_source":false,"parameter_size":"","provider":"openai","url":""},"gpt4all-13b-snoozy":{"country":"US","id":"gpt4all-13b-snoozy","launched":"","modalities":["text"],"model":"GPT4All 13B","model_family":"GPT4All","model_variant":"GPT4All 13B Snoozy","model_version":"GPT4All","name":"GPT4All 13B Snoozy","open_source":true,"parameter_size":"13B","provider":"nomic","url":""},"granite-13b":{"country":"US","id":"granite-13b","launched":"","modalities":["text"],"model":"Granite 13B","model_family":"Granite","model_variant":"Granite 13B","model_version":"Granite 3.1","name":"Granite 13B","open_source":true,"parameter_size":"13B","provider":"IBM","url":""},"granite-3.0-2b-instruct":{"country":"US","id":"granite-3.0-2b-instruct","launched":"","modalities":["text"],"model":"Granite 3.0 2B","model_family":"Granite","model_variant":"3.0 2B Instruct","model_version":"Granite 3.0","name":"Granite 3.0 2B Instruct","open_source":true,"parameter_size":"2B","provider":"IBM","url":""},"granite-3.0-8b-instruct":{"country":"US","id":"granite-3.0-8b-instruct","launched":"2024-10-21","modalities":["text"],"model":"Granite 3.0 8B","model_family":"Granite","model_variant":"Granite 3.0 8B Instruct","model_version":"Granite 3.1","name":"Granite 3.0 8B Instruct","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-3.1-2b-instruct":{"country":"US","id":"granite-3.1-2b-instruct","launched":"2024-12-18","modalities":["text"],"model":"Granite 3.1 2B","model_family":"Granite","model_variant":"Granite 3.1 2B Instruct","model_version":"Granite 3.1","name":"Granite 3.1 2B Instruct","open_source":true,"parameter_size":"2B","provider":"IBM","url":""},"granite-3.1-8b-instruct":{"country":"US","id":"granite-3.1-8b-instruct","launched":"2024-12-18","modalities":["text"],"model":"Granite 3.1 8B","model_family":"Granite","model_variant":"Granite 3.1 8B Instruct","model_version":"Granite 3.1","name":"Granite 3.1 8B Instruct","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-3.2-2b-instruct":{"country":"US","id":"granite-3.2-2b-instruct","launched":"2025-02-26","modalities":["text"],"model":"Granite 3.2 2B","model_family":"Granite","model_variant":"Granite 3.2 2B Instruct","model_version":"Granite 3.2","name":"Granite 3.2 2B Instruct","open_source":true,"parameter_size":"2B","provider":"IBM","url":""},"granite-3.2-8b-instruct":{"country":"US","id":"granite-3.2-8b-instruct","launched":"2025-02-26","modalities":["text"],"model":"Granite 3.2 8B","model_family":"Granite","model_variant":"Granite 3.2 8B Instruct","model_version":"Granite 3.2","name":"Granite 3.2 8B Instruct","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-3.3-2b-instruct":{"country":"US","id":"granite-3.3-2b-instruct","launched":"2025-04-16","modalities":["text"],"model":"Granite 3.3 2B","model_family":"Granite","model_variant":"Granite 3.3 2B Instruct","model_version":"Granite 3.3","name":"Granite 3.3 2B Instruct","open_source":true,"parameter_size":"2B","provider":"IBM","url":""},"granite-3.3-8b-base":{"country":"US","id":"granite-3.3-8b-base","launched":"","modalities":["text"],"model":"Granite 3.3 8B","model_family":"Granite","model_variant":"Granite 3.3 8B","model_version":"Granite 3.3","name":"Granite 3.3 8B Base","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-3.3-8b-instruct":{"country":"US","id":"granite-3.3-8b-instruct","launched":"2025-04-16","modalities":["text"],"model":"Granite 3.3 8B","model_family":"Granite","model_variant":"Granite 3.3 8B Instruct","model_version":"Granite 3.3","name":"Granite 3.3 8B Instruct","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-4-300m":{"country":"US","id":"granite-4-300m","launched":"2025-10-28","modalities":["text"],"model":"Granite 4 300M","model_family":"Granite","model_variant":"Granite-4-300M","model_version":"Granite 4","name":"Granite-4-300M","open_source":true,"parameter_size":"0.3B","provider":"IBM","url":""},"granite-4-h-300m":{"country":"US","id":"granite-4-h-300m","launched":"2025-10-28","modalities":["text"],"model":"Granite 4 H 300M","model_family":"Granite","model_variant":"Granite-4-H-300M","model_version":"Granite 4","name":"Granite-4-H-300M","open_source":true,"parameter_size":"0.3B","provider":"IBM","url":""},"granite-4.0-1b":{"country":"US","id":"granite-4.0-1b","launched":"2025-11-17","modalities":["text"],"model":"Granite 4.0 1B","model_family":"Granite","model_variant":"Granite 4.0 1B","model_version":"Granite 4.0","name":"Granite-4.0-1B","open_source":true,"parameter_size":"1B","provider":"IBM","url":""},"granite-4.0-h-1b":{"country":"US","id":"granite-4.0-h-1b","launched":"","modalities":["text"],"model":"Granite 4.0 H 1B","model_family":"Granite","model_variant":"Granite 4.0 H 1B","model_version":"Granite 4.0","name":"Granite-4.0-H-1B","open_source":true,"parameter_size":"1B","provider":"IBM","url":""},"granite-4.0-h-micro":{"country":"US","id":"granite-4.0-h-micro","launched":"2025-10-02","modalities":["text"],"model":"Granite 4.0 H Micro","model_family":"Granite","model_variant":"Granite 4.0-H-Micro","model_version":"Granite 4.0","name":"Granite-4.0-H-Micro","open_source":true,"parameter_size":"3.19B","provider":"IBM","url":""},"granite-4.0-h-small":{"country":"US","id":"granite-4.0-h-small","launched":"2025-10-02","modalities":["text"],"model":"Granite 4.0 H Small","model_family":"Granite","model_variant":"Granite 4.0-H-Small","model_version":"Granite 4.0","name":"Granite-4.0-H-Small","open_source":true,"parameter_size":"32.2B","provider":"IBM","url":""},"granite-4.0-h-small-base":{"country":"US","id":"granite-4.0-h-small-base","launched":"2025-10-02","modalities":["text"],"model":"Granite 4.0 H Small","model_family":"Granite","model_variant":"Granite 4.0-H-Small-Base","model_version":"Granite 4.0","name":"Granite-4.0-H-Small-Base","open_source":true,"parameter_size":"32.2B","provider":"IBM","url":""},"granite-4.0-h-tiny":{"country":"US","id":"granite-4.0-h-tiny","launched":"2025-10-02","modalities":["text"],"model":"Granite 4.0 H Tiny","model_family":"Granite","model_variant":"Granite 4.0-H-Tiny","model_version":"Granite 4.0","name":"Granite-4.0-H-Tiny","open_source":true,"parameter_size":"6.94B","provider":"IBM","url":""},"granite-4.0-micro":{"country":"US","id":"granite-4.0-micro","launched":"2025-10-02","modalities":["text"],"model":"Granite 4.0 Micro","model_family":"Granite","model_variant":"Granite 4.0-Micro","model_version":"Granite 4.0","name":"Granite-4.0-Micro","open_source":true,"parameter_size":"3.4B","provider":"IBM","url":""},"granite-4.0-tiny-preview":{"country":"US","id":"granite-4.0-tiny-preview","launched":"2025-05-02","modalities":["text"],"model":"Granite 4.0 Tiny Preview","model_family":"Granite","model_variant":"Granite 4.0-Tiny-Preview","model_version":"Granite 4.0","name":"Granite-4.0-Tiny-Preview","open_source":true,"parameter_size":"6.67B","provider":"IBM","url":""},"granite-7b":{"country":"US","id":"granite-7b","launched":"","modalities":["text"],"model":"Granite 7B","model_family":"Granite","model_variant":"Granite 7B","model_version":"Granite 3.1","name":"Granite 7B","open_source":true,"parameter_size":"7B","provider":"IBM","url":""},"granite-c-8b":{"country":"US","id":"granite-c-8b","launched":"","modalities":["text"],"model":"Granite C 8B","model_family":"Granite","model_variant":"Granite C 8B","model_version":"Granite 3.1","name":"Granite C 8B","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-c-instruct-8b":{"country":"US","id":"granite-c-instruct-8b","launched":"","modalities":["text"],"model":"Granite C Instruct 8B","model_family":"Granite","model_variant":"Granite C Instruct 8B","model_version":"Granite 3.1","name":"Granite C Instruct 8B","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-code-20b-base":{"country":"US","id":"granite-code-20b-base","launched":"","modalities":["text"],"model":"Granite Code 20B","model_family":"Granite","model_variant":"20B Base","model_version":"Granite Code","name":"Granite Code 20B Base","open_source":true,"parameter_size":"20B","provider":"IBM","url":""},"granite-code-20b-instruct":{"country":"US","id":"granite-code-20b-instruct","launched":"","modalities":["text"],"model":"Granite Code 20B","model_family":"Granite","model_variant":"Granite Code 20B Instruct","model_version":"Granite 3.1","name":"Granite Code 20B Instruct","open_source":true,"parameter_size":"20B","provider":"IBM","url":""},"granite-code-34b-base":{"country":"US","id":"granite-code-34b-base","launched":"","modalities":["text"],"model":"Granite Code 34B","model_family":"Granite","model_variant":"34B Base","model_version":"Granite Code","name":"Granite Code 34B Base","open_source":true,"parameter_size":"34B","provider":"IBM","url":""},"granite-code-34b-instruct":{"country":"US","id":"granite-code-34b-instruct","launched":"","modalities":["text"],"model":"Granite Code 34B","model_family":"Granite","model_variant":"Granite Code 34B Instruct","model_version":"Granite 3.1","name":"Granite Code 34B Instruct","open_source":true,"parameter_size":"34B","provider":"IBM","url":""},"granite-code-8b-base":{"country":"US","id":"granite-code-8b-base","launched":"","modalities":["text"],"model":"Granite Code 8B","model_family":"Granite","model_variant":"8B Base","model_version":"Granite Code","name":"Granite Code 8B Base","open_source":true,"parameter_size":"8B","provider":"IBM","url":""},"granite-docling-258m":{"country":"US","id":"granite-docling-258m","launched":"","modalities":["visual"],"model":"Granite Docling 258M","model_family":"Granite","model_variant":"Granite Docling 258M","model_version":"Granite Docling","name":"Granite Docling 258M","open_source":true,"parameter_size":"0.258B","provider":"IBM","url":""},"grok-2":{"country":"US","id":"grok-2","launched":"2024-08-14","modalities":["text"],"model":"Grok 2","model_family":"Grok","model_variant":"Grok 2","model_version":"Grok 2","name":"Grok 2","open_source":true,"parameter_size":"270Be","provider":"xai","url":""},"grok-2-1212":{"country":"US","id":"grok-2-1212","launched":"2024-08-14","modalities":["text"],"model":"Grok 2","model_family":"Grok","model_variant":"Grok 2","model_version":"Grok 2","name":"Grok 2 (2024-12-12)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-2-2024-08-13":{"country":"US","id":"grok-2-2024-08-13","launched":"2024-08-14","modalities":["text"],"model":"Grok 2","model_family":"Grok","model_variant":"Grok-2","model_version":"Grok 2","name":"Grok-2 (2024-08-13)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-2-mini":{"country":"US","id":"grok-2-mini","launched":"2024-08-14","modalities":["text"],"model":"Grok 2 Mini","model_family":"Grok","model_variant":"Grok 2 Mini","model_version":"Grok 2","name":"Grok 2 Mini","open_source":false,"parameter_size":"100Be","provider":"xai","url":""},"grok-2-mini-2024-08-13":{"country":"US","id":"grok-2-mini-2024-08-13","launched":"2024-08-14","modalities":["text"],"model":"Grok 2 Mini","model_family":"Grok","model_variant":"Grok-2 Mini","model_version":"Grok 2","name":"Grok-2 Mini (2024-08-13)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3":{"country":"US","id":"grok-3","launched":"2025-02-17","modalities":["text"],"model":"Grok 3","model_family":"Grok","model_variant":"Grok 3","model_version":"Grok 3","name":"Grok 3","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-beta":{"country":"US","id":"grok-3-beta","launched":"2025-02-17","modalities":["text"],"model":"Grok 3","model_family":"Grok","model_variant":"Grok 3","model_version":"Grok 3","name":"Grok 3 Beta","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-beta-think":{"country":"US","id":"grok-3-beta-think","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Think","model_family":"Grok","model_variant":"Grok 3","model_version":"Grok 3","name":"Grok 3 Think","open_source":false,"parameter_size":"2700Be","provider":"xai","url":""},"grok-3-mini":{"country":"US","id":"grok-3-mini","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Mini","model_family":"Grok","model_variant":"Grok 3","model_version":"Grok 3","name":"Grok 3 Mini","open_source":false,"parameter_size":"100Be","provider":"xai","url":""},"grok-3-mini-beta":{"country":"US","id":"grok-3-mini-beta","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Mini","model_family":"Grok","model_variant":"Grok 3 Mini Beta","model_version":"Grok 3","name":"Grok-3-Mini-Beta","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-mini-beta-high":{"country":"US","id":"grok-3-mini-beta-high","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Mini","model_family":"Grok","model_variant":"Grok 3 Mini Beta High","model_version":"Grok 3","name":"Grok 3 Mini Beta (High)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-mini-beta-low":{"country":"US","id":"grok-3-mini-beta-low","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Mini","model_family":"Grok","model_variant":"Grok 3 Mini Beta Low","model_version":"Grok 3","name":"Grok 3 Mini Beta (Low)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-mini-fast-high-reasoning":{"country":"US","id":"grok-3-mini-fast-high-reasoning","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Mini","model_family":"Grok","model_variant":"Grok 3 Mini Fast High","model_version":"Grok 3","name":"Grok 3 Mini Fast High Reasoning","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-mini-fast-low-reasoning":{"country":"US","id":"grok-3-mini-fast-low-reasoning","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Mini","model_family":"Grok","model_variant":"Grok 3 Mini Fast Low","model_version":"Grok 3","name":"Grok 3 Mini Fast Low Reasoning","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-mini-high":{"country":"US","id":"grok-3-mini-high","launched":"2025-02-17","modalities":["text"],"model":"Grok 3 Mini","model_family":"Grok","model_variant":"Grok 3 Mini High","model_version":"Grok 3","name":"Grok 3 Mini (High)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-3-preview-02-24":{"country":"US","id":"grok-3-preview-02-24","launched":"2025-02-17","modalities":["text"],"model":"Grok 3","model_family":"Grok","model_variant":"Grok-3 Preview","model_version":"Grok 3","name":"Grok-3 Preview (02-24)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-4":{"country":"US","id":"grok-4","launched":"2025-07-09","modalities":["text"],"model":"Grok 4","model_family":"Grok","model_variant":"Grok 4","model_version":"Grok 4","name":"Grok 4","open_source":false,"parameter_size":"1700Be","provider":"xai","url":""},"grok-4-0709":{"country":"US","id":"grok-4-0709","launched":"2025-07-09","modalities":["text"],"model":"Grok 4","model_family":"Grok","model_variant":"Grok 4","model_version":"Grok 4","name":"Grok 4 (2024-07-09)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-4-fast":{"country":"US","id":"grok-4-fast","launched":"2025-07-09","modalities":["text"],"model":"Grok 4 Fast","model_family":"Grok","model_variant":"Grok 4 Fast","model_version":"Grok 4","name":"Grok 4 Fast","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-4-fast-thinking":{"country":"US","id":"grok-4-fast-thinking","launched":"2025-07-09","modalities":["text"],"model":"Grok 4 Fast","model_family":"Grok","model_variant":"Grok 4 Fast Thinking","model_version":"Grok 4","name":"Grok 4 Fast Thinking","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-4-high":{"country":"US","id":"grok-4-high","launched":"2025-07-09","modalities":["text"],"model":"Grok 4","model_family":"Grok","model_variant":"Grok 4 High","model_version":"Grok 4","name":"Grok 4 (High)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-4-thinking":{"country":"US","id":"grok-4-thinking","launched":"2025-07-09","modalities":["text"],"model":"Grok 4","model_family":"Grok","model_variant":"Grok 4 Thinking","model_version":"Grok 4","name":"Grok 4 (Thinking)","open_source":false,"parameter_size":"1700Be","provider":"xai","url":""},"grok-4.1-fast":{"country":"US","id":"grok-4.1-fast","launched":"2025-11-18","modalities":["text"],"model":"Grok 4.1","model_family":"Grok","model_variant":"Grok 4.1 Fast","model_version":"Grok 4.1","name":"Grok 4.1 Fast","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-4.1-thinking":{"country":"US","id":"grok-4.1-thinking","launched":"2025-11-18","modalities":["text"],"model":"Grok 4.1","model_family":"Grok","model_variant":"Grok 4.1 Thinking","model_version":"Grok 4.1","name":"Grok 4.1 (Thinking)","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-beta":{"country":"US","id":"grok-beta","launched":"","modalities":["text"],"model":"Grok Beta","model_family":"Grok","model_variant":"Grok Beta","model_version":"Grok","name":"Grok Beta","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-code-fast":{"country":"US","id":"grok-code-fast","launched":"","modalities":["text"],"model":"Grok Code Fast","model_family":"Grok","model_variant":"Grok Code Fast","model_version":"Grok Code","name":"Grok Code Fast","open_source":false,"parameter_size":"","provider":"xai","url":""},"grok-code-fast-1":{"country":"US","id":"grok-code-fast-1","launched":"2025-08-25","modalities":["text"],"model":"Grok Code Fast 1","model_family":"Grok","model_variant":"Code Fast 1","model_version":"Grok Code","name":"Grok Code Fast 1","open_source":false,"parameter_size":"50Be","provider":"xai","url":""},"grok-core-poset":{"country":"US","id":"grok-core-poset","launched":"","modalities":["text"],"model":"Grok Core","model_family":"Grok","model_variant":"Core Poset","model_version":"Grok Core","name":"Grok Core Poset","open_source":false,"parameter_size":"","provider":"xai","url":""},"gta1-32b":{"country":"US","id":"gta1-32b","launched":"","modalities":["visual"],"model":"GTA1 32B","model_family":"GTA1","model_variant":"GTA1-32B","model_version":"GTA1","name":"GTA1 32B","open_source":true,"parameter_size":"32B","provider":"salesforce","url":""},"gta1-72b":{"country":"US","id":"gta1-72b","launched":"","modalities":["visual"],"model":"GTA1 72B","model_family":"GTA1","model_variant":"GTA1-72B","model_version":"GTA1","name":"GTA1 72B","open_source":true,"parameter_size":"72B","provider":"salesforce","url":""},"guanaco-13b":{"country":"US","id":"guanaco-13b","launched":"","modalities":["text"],"model":"Guanaco 13B","model_family":"Guanaco","model_variant":"Guanaco-13B","model_version":"Guanaco","name":"Guanaco-13B","open_source":true,"parameter_size":"13B","provider":"timdettmers","url":""},"guanaco-33b":{"country":"US","id":"guanaco-33b","launched":"","modalities":["text"],"model":"Guanaco","model_family":"Guanaco","model_variant":"Guanaco-33B","model_version":"Guanaco-33B","name":"Guanaco-33B","open_source":true,"parameter_size":"33B","provider":"timdettmers","url":""},"guanaco-65b":{"country":"US","id":"guanaco-65b","launched":"","modalities":["text"],"model":"Guanaco 65B","model_family":"Guanaco","model_variant":"Guanaco-65B","model_version":"Guanaco","name":"Guanaco-65B","open_source":true,"parameter_size":"65B","provider":"timdettmers","url":""},"guanaco-7b":{"country":"US","id":"guanaco-7b","launched":"","modalities":["text"],"model":"Guanaco 7B","model_family":"Guanaco","model_variant":"Guanaco-7B","model_version":"Guanaco","name":"Guanaco-7B","open_source":true,"parameter_size":"7B","provider":"timdettmers","url":""},"gundam":{"country":"CN","id":"gundam","launched":"2025-10-20","modalities":["visual"],"model":"DeepSeek-OCR Gundam","model_family":"DeepSeek OCR","model_variant":"Base","model_version":"DeepSeek OCR","name":"Gundam","open_source":true,"parameter_size":"3B","provider":"deepseek","url":""},"gundam-m":{"country":"CN","id":"gundam-m","launched":"2025-10-20","modalities":["visual"],"model":"DeepSeek-OCR Gundam-M","model_family":"DeepSeek OCR","model_variant":"Medium","model_version":"DeepSeek OCR","name":"Gundam-M","open_source":true,"parameter_size":"3B","provider":"deepseek","url":""},"hairuo":{"country":"CN","id":"hairuo","launched":"","modalities":["text"],"model":"Hairuo","model_family":"Hairuo","model_variant":"Hairuo","model_version":"Hairuo","name":"Hairuo","open_source":true,"parameter_size":"200Be","provider":"generic","url":""},"harmon-1.5b":{"country":"US","id":"harmon-1.5b","launched":"","modalities":["visual"],"model":"Harmon","model_family":"Harmon","model_variant":"Harmon-1.5B","model_version":"Harmon-1.5B","name":"Harmon-1.5B","open_source":true,"parameter_size":"1.5B","provider":"harmonai","url":""},"hcx":{"country":"US","id":"hcx","launched":"","modalities":["text"],"model":"HCX","model_family":"HCX","model_variant":"HCX","model_version":"HCX","name":"HCX","open_source":false,"parameter_size":"","provider":"hcx","url":""},"hcx-dash":{"country":"US","id":"hcx-dash","launched":"","modalities":["text"],"model":"HCX Dash","model_family":"HCX","model_variant":"HCX Dash","model_version":"HCX","name":"HCX Dash","open_source":false,"parameter_size":"","provider":"hcx","url":""},"hermes-2-pro-llama-3-70b":{"country":"US","id":"hermes-2-pro-llama-3-70b","launched":"","modalities":["text"],"model":"Hermes 2 PRO Llama 3.70B","model_family":"Hermes","model_variant":"Hermes 2 PRO Llama 3.70B","model_version":"Hermes 3","name":"Hermes 2 PRO Llama 3.70B","open_source":true,"parameter_size":"3.70B","provider":"nousresearch","url":""},"hermes-2-pro-llama-3.70b":{"country":"US","id":"hermes-2-pro-llama-3.70b","launched":"","modalities":["text"],"model":"Hermes 2 PRO Llama 3.70B","model_family":"Hermes","model_variant":"Hermes 2 PRO Llama 3.70B","model_version":"Hermes 3","name":"Hermes 2 PRO Llama 3.70B","open_source":true,"parameter_size":"3.70B","provider":"nousresearch","url":""},"hermes-2-theta-llama-3-70b":{"country":"US","id":"hermes-2-theta-llama-3-70b","launched":"","modalities":["text"],"model":"Hermes 2 Theta Llama 3.70B","model_family":"Hermes","model_variant":"Hermes 2 Theta Llama 3.70B","model_version":"Hermes 3","name":"Hermes 2 Theta Llama 3.70B","open_source":true,"parameter_size":"3.70B","provider":"nousresearch","url":""},"hermes-2-theta-llama-3.70b":{"country":"US","id":"hermes-2-theta-llama-3.70b","launched":"","modalities":["text"],"model":"Hermes 2 Theta Llama 3.70B","model_family":"Hermes","model_variant":"Hermes 2 Theta Llama 3.70B","model_version":"Hermes 3","name":"Hermes 2 Theta Llama 3.70B","open_source":true,"parameter_size":"3.70B","provider":"nousresearch","url":""},"hermes-3-llama-3.1-405b":{"country":"US","id":"hermes-3-llama-3.1-405b","launched":"2024-08-16","modalities":["text"],"model":"Hermes 3 Llama-3.1 405B","model_family":"Hermes","model_variant":"Hermes-3","model_version":"Hermes 3","name":"Hermes-3-Llama-3.1-405B","open_source":true,"parameter_size":"405B","provider":"nousresearch","url":""},"hermes-3-llama-3.1-70b":{"country":"US","id":"hermes-3-llama-3.1-70b","launched":"2024-08-16","modalities":["text"],"model":"Hermes 3 Llama-3.1 70B","model_family":"Hermes","model_variant":"Hermes-3","model_version":"Hermes 3","name":"Hermes-3-Llama-3.1-70B","open_source":true,"parameter_size":"70B","provider":"nousresearch","url":""},"hermes-4-405b":{"country":"US","id":"hermes-4-405b","launched":"","modalities":["text"],"model":"Hermes 4 405B","model_family":"Hermes","model_variant":"Hermes-4","model_version":"Hermes 4","name":"Hermes-4-405B","open_source":true,"parameter_size":"405B","provider":"nousresearch","url":""},"hermes-4-70b":{"country":"US","id":"hermes-4-70b","launched":"2025-08-27","modalities":["text"],"model":"Hermes 4 70B","model_family":"Hermes","model_variant":"Hermes-70B","model_version":"Hermes 4","name":"Hermes 4 70B","open_source":true,"parameter_size":"70B","provider":"nousresearch","url":""},"hermes-4.3-36b":{"country":"US","id":"hermes-4.3-36b","launched":"2025-12-03","modalities":["text"],"model":"Hermes 4.3 36B","model_family":"Hermes","model_variant":"Hermes 4.3 36B","model_version":"Hermes 4.3","name":"Hermes 4.3 36B","open_source":true,"parameter_size":"36B","provider":"nousresearch","url":""},"hermes-4.3-36b-pyche":{"country":"US","id":"hermes-4.3-36b-pyche","launched":"2025-12-03","modalities":["text"],"model":"Hermes 4.3 36B Pyche","model_family":"Hermes","model_variant":"Hermes 4.3 36B Pyche","model_version":"Hermes 4.3","name":"Hermes 4.3 36B Pyche","open_source":true,"parameter_size":"36B","provider":"nousresearch","url":""},"hermes-4.3-70b":{"country":"US","id":"hermes-4.3-70b","launched":"2025-12-03","modalities":["text"],"model":"Hermes 4.3 70B","model_family":"Hermes","model_variant":"Hermes 4.3 70B","model_version":"Hermes 4.3","name":"Hermes 4.3 70B","open_source":true,"parameter_size":"70B","provider":"nousresearch","url":""},"holo1-3b":{"country":"FR","id":"holo1-3b","launched":"","modalities":["visual"],"model":"Holo1-3B","model_family":"Holo","model_variant":"Holo1-3B","model_version":"Holo1","name":"Holo1-3B","open_source":true,"parameter_size":"3B","provider":"hcompany","url":""},"holo1-7b":{"country":"FR","id":"holo1-7b","launched":"","modalities":["visual"],"model":"Holo1-7B","model_family":"Holo","model_variant":"Holo1-7B","model_version":"Holo1","name":"Holo1-7B","open_source":true,"parameter_size":"7B","provider":"hcompany","url":""},"holo1.5-3b":{"country":"FR","id":"holo1.5-3b","launched":"","modalities":["visual"],"model":"Holo1.5-3B","model_family":"Holo","model_variant":"Holo1.5-3B","model_version":"Holo1.5","name":"Holo1.5-3B","open_source":true,"parameter_size":"3B","provider":"hcompany","url":""},"holo1.5-72b":{"country":"FR","id":"holo1.5-72b","launched":"","modalities":["visual"],"model":"Holo1.5-72B","model_family":"Holo","model_variant":"Holo1.5-72B","model_version":"Holo1.5","name":"Holo1.5-72B","open_source":true,"parameter_size":"72B","provider":"hcompany","url":""},"holo1.5-7b":{"country":"FR","id":"holo1.5-7b","launched":"","modalities":["visual"],"model":"Holo1.5-7B","model_family":"Holo","model_variant":"Holo1.5-7B","model_version":"Holo1.5","name":"Holo1.5-7B","open_source":true,"parameter_size":"7B","provider":"hcompany","url":""},"hunyuan-4b-instruct":{"country":"CN","id":"hunyuan-4b-instruct","launched":"","modalities":["text"],"model":"Hunyuan 4B","model_family":"Hunyuan","model_variant":"Hunyuan-4b-instruct-4B","model_version":"Hunyuan","name":"Hunyuan-4B-Instruct","open_source":true,"parameter_size":"4B","provider":"tencent","url":""},"hunyuan-7b-instruct":{"country":"CN","id":"hunyuan-7b-instruct","launched":"","modalities":["text"],"model":"Hunyuan 7B","model_family":"Hunyuan","model_variant":"Hunyuan-7b-instruct-7B","model_version":"Hunyuan","name":"Hunyuan-7B-Instruct","open_source":true,"parameter_size":"7B","provider":"tencent","url":""},"hunyuan-7b-pretrain":{"country":"CN","id":"hunyuan-7b-pretrain","launched":"","modalities":["text"],"model":"Hunyuan 7B","model_family":"Hunyuan","model_variant":"Hunyuan-7b-pretrain-7B","model_version":"Hunyuan","name":"Hunyuan-7B-Pretrain","open_source":true,"parameter_size":"7B","provider":"tencent","url":""},"hunyuan-80b-a13b-instruct":{"active_parameters":"13B","country":"CN","id":"hunyuan-80b-a13b-instruct","launched":"","modalities":["text"],"model":"Hunyuan 80B A13B","model_family":"Hunyuan","model_variant":"Hunyuan-80B-A13B-Instruct","model_version":"Hunyuan","name":"Hunyuan-80B-A13B-Instruct","open_source":true,"parameter_size":"80B","provider":"tencent","url":""},"hunyuan-coder-b8-preview":{"country":"CN","id":"hunyuan-coder-b8-preview","launched":"","modalities":["text"],"model":"Hunyuan Coder B8","model_family":"Hunyuan","model_variant":"Hunyuan Coder B8 Preview","model_version":"Hunyuan","name":"Hunyuan Coder B8 Preview","open_source":true,"parameter_size":"8B","provider":"tencent","url":""},"hunyuan-large-2025-02-10":{"country":"CN","id":"hunyuan-large-2025-02-10","launched":"2025-02-10","modalities":["text"],"model":"Hunyuan Large","model_family":"Hunyuan","model_variant":"Hunyuan Large 2025-02-10","model_version":"Hunyuan","name":"Hunyuan Large 2025-02-10","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hunyuan-large-vision":{"country":"CN","id":"hunyuan-large-vision","launched":"","modalities":["visual"],"model":"Hunyuan Large Vision","model_family":"Hunyuan","model_variant":"Hunyuan Large Vision","model_version":"Hunyuan Vision","name":"Hunyuan Large Vision","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hunyuan-standard-2025-02-10":{"country":"CN","id":"hunyuan-standard-2025-02-10","launched":"2025-02-10","modalities":["text"],"model":"Hunyuan Standard","model_family":"Hunyuan","model_variant":"Hunyuan Standard 2025-02-10","model_version":"Hunyuan","name":"Hunyuan Standard 2025-02-10","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hunyuan-standard-256k":{"country":"CN","id":"hunyuan-standard-256k","launched":"","modalities":["text"],"model":"Hunyuan Standard","model_family":"Hunyuan","model_variant":"Hunyuan Standard 256K","model_version":"Hunyuan","name":"Hunyuan Standard 256K","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hunyuan-standard-vision-2024-12-31":{"country":"CN","id":"hunyuan-standard-vision-2024-12-31","launched":"2024-12-31","modalities":["visual"],"model":"Hunyuan Standard Vision","model_family":"Hunyuan","model_variant":"Standard Vision 20241231","model_version":"Hunyuan Vision","name":"Hunyuan Standard Vision (2024-12-31)","open_source":false,"parameter_size":"","provider":"tencent","url":"https://lmarena.ai/leaderboard/vision"},"hunyuan-t1":{"country":"CN","id":"hunyuan-t1","launched":"2025-03-24","modalities":["text"],"model":"Hunyuan-T1","model_family":"Hunyuan","model_variant":"HunyuanT1","model_version":"Hunyuan T1","name":"Hunyuan-T1","open_source":true,"parameter_size":"389B","provider":"tencent","url":""},"hunyuan-t1-20250711":{"country":"CN","id":"hunyuan-t1-20250711","launched":"2025-07-11","modalities":["text"],"model":"Hunyuan-T1","model_family":"Hunyuan","model_variant":"Hunyuan T1 20250711","model_version":"Hunyuan T1","name":"Hunyuan-T1 2025-07-11","open_source":true,"parameter_size":"389B","provider":"tencent","url":""},"hunyuan-turbo-0110":{"country":"CN","id":"hunyuan-turbo-0110","launched":"2025-01-10","modalities":["text"],"model":"Hunyuan Turbo","model_family":"Hunyuan","model_variant":"Hunyuan Turbo 0110","model_version":"Hunyuan Turbo","name":"Hunyuan Turbo 0110","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hunyuan-turbos":{"country":"CN","id":"hunyuan-turbos","launched":"","modalities":["text"],"model":"Hunyuan Turbos","model_family":"Hunyuan","model_variant":"Hunyuan Turbos","model_version":"Hunyuan Turbos","name":"Hunyuan Turbos","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hunyuan-turbos-20250226":{"country":"CN","id":"hunyuan-turbos-20250226","launched":"2025-02-26","modalities":["text"],"model":"Hunyuan Turbos","model_family":"Hunyuan","model_variant":"Hunyuan Turbos 20250226","model_version":"Hunyuan Turbos","name":"Hunyuan Turbos 2025-02-26","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hunyuan-turbos-20250416":{"country":"CN","id":"hunyuan-turbos-20250416","launched":"2025-04-16","modalities":["text"],"model":"Hunyuan Turbos","model_family":"Hunyuan","model_variant":"Hunyuan Turbos 20250416","model_version":"Hunyuan Turbos","name":"Hunyuan Turbos 2025-04-16","open_source":true,"parameter_size":"","provider":"tencent","url":""},"hy-0.5b":{"country":"CN","id":"hy-0.5b","launched":"2026-02-04","modalities":["text"],"model":"HY 0.5B","model_family":"HY","model_variant":"HY 0.5B","model_version":"HY","name":"HY-0.5B","open_source":true,"parameter_size":"0.5B","provider":"tencent","url":""},"hy-1.8b":{"country":"CN","id":"hy-1.8b","launched":"2026-02-04","modalities":["text"],"model":"HY 1.8B","model_family":"HY","model_variant":"HY 1.8B","model_version":"HY","name":"HY-1.8B","open_source":true,"parameter_size":"1.8B","provider":"tencent","url":""},"hy-1.8b-2bit":{"country":"CN","id":"hy-1.8b-2bit","launched":"2026-02-04","modalities":["text"],"model":"HY 1.8B 2Bit","model_family":"HY","model_variant":"HY 1.8B 2Bit","model_version":"HY","name":"HY-1.8B-2Bit","open_source":true,"parameter_size":"1.8B","provider":"tencent","url":""},"hy-1.8b-int4gptq":{"country":"CN","id":"hy-1.8b-int4gptq","launched":"2026-02-04","modalities":["text"],"model":"HY 1.8B int4gptq","model_family":"HY","model_variant":"HY 1.8B int4gptq","model_version":"HY","name":"HY-1.8B-int4gptq","open_source":true,"parameter_size":"1.8B","provider":"tencent","url":""},"hyperclovax-seed-text-instruct-1.5b":{"country":"KR","id":"hyperclovax-seed-text-instruct-1.5b","launched":"","modalities":["text"],"model":"HyperCLOVA X SEED Text Instruct 1.5B","model_family":"HyperCLOVA X","model_variant":"SEED Text Instruct 1.5B","model_version":"HyperCLOVA X","name":"HyperCLOVA X SEED Text Instruct 1.5B","open_source":true,"parameter_size":"1.5B","provider":"naver","url":""},"hyperclovax-seed-think":{"country":"KR","id":"hyperclovax-seed-think","launched":"2025-12-29","modalities":["text"],"model":"HyperCLOVA X SEED Think","model_family":"HyperCLOVA X","model_variant":"SEED Think","model_version":"HyperCLOVA X","name":"HyperCLOVA X SEED Think","open_source":true,"parameter_size":"32B","provider":"naver","url":""},"icecuber":{"country":"US","id":"icecuber","launched":"2024-04-01","modalities":["text"],"model":"Icecuber","model_family":"ARChitects","model_variant":"Icecuber","model_version":"ARChitects","name":"Icecuber","open_source":false,"parameter_size":"","provider":"arcprize","url":""},"incoder-1.3b":{"country":"IL","id":"incoder-1.3b","launched":"","modalities":["text"],"model":"InCoder 1.3B","model_family":"InCoder","model_variant":"InCoder-1.3B","model_version":"InCoder","name":"InCoder-1.3B","open_source":true,"parameter_size":"1.3B","provider":"ai21","url":""},"incoder-6.7b":{"country":"IL","id":"incoder-6.7b","launched":"","modalities":["text"],"model":"InCoder 6.7B","model_family":"InCoder","model_variant":"InCoder-6.7B","model_version":"InCoder","name":"InCoder-6.7B","open_source":true,"parameter_size":"6.7B","provider":"ai21","url":""},"infinity-parser-7b":{"country":"CN","id":"infinity-parser-7b","launched":"","modalities":["visual"],"model":"Infinity-Parser 7B","model_family":"Infinity-Parser","model_variant":"7B","model_version":"Infinity-Parser","name":"Infinity-Parser 7B","open_source":true,"parameter_size":"7B","provider":"infly","url":""},"intellect-3":{"active_parameters":"12B","country":"US","id":"intellect-3","launched":"2025-11-27","modalities":["text"],"model":"INTELLECT-3","model_family":"INTELLECT","model_variant":"INTELLECT 3","model_version":"INTELLECT","name":"INTELLECT-3","open_source":true,"parameter_size":"107B","provider":"primeintellect","url":""},"intern-s1-pro":{"active_parameters":"22B","country":"CN","id":"intern-s1-pro","launched":"2026-02-02","modalities":["text","visual"],"model":"Intern-S1-Pro","model_family":"Intern","model_variant":"Intern-S1-Pro","model_version":"Intern-S1","name":"Intern-S1-Pro","open_source":true,"parameter_size":"1000B","provider":"internlm","url":"https://huggingface.co/internlm/Intern-S1-Pro"},"internlm2.5-7b-chat":{"country":"CN","id":"internlm2.5-7b-chat","launched":"","modalities":["text"],"model":"Internlm2.5 7B Chat","model_family":"Intern","model_variant":"Internlm2.5 7B Chat","model_version":"Internlm2.5","name":"Internlm2.5 7B Chat","open_source":true,"parameter_size":"7B","provider":"internlm","url":""},"internlm2.5.7b-chat":{"country":"CN","id":"internlm2.5.7b-chat","launched":"","modalities":["text"],"model":"Internlm2 5.7B Chat","model_family":"Intern","model_variant":"Internlm2 5.7B Chat","model_version":"Internlm2.5.7b","name":"Internlm2 5.7B Chat","open_source":true,"parameter_size":"5.7B","provider":"internlm","url":""},"internlm2_5-20b-chat":{"country":"CN","id":"internlm2_5-20b-chat","launched":"","modalities":["text"],"model":"InternLM2.5 20B Chat","model_family":"Intern","model_variant":"InternLM2.5 20B Chat","model_version":"InternLM2.5","name":"InternLM2.5 20B Chat","open_source":true,"parameter_size":"20B","provider":"internlm","url":""},"internvl-8b":{"country":"CN","id":"internvl-8b","launched":"2025-09-28","modalities":["visual"],"model":"InternVL 8B","model_family":"Intern","model_variant":"InternVL-8B","model_version":"InternVL","name":"InternVL-8B","open_source":true,"parameter_size":"8B","provider":"opengvlab","url":""},"internvl2-26b":{"country":"CN","id":"internvl2-26b","launched":"","modalities":["visual"],"model":"InternVL2 26B","model_family":"Intern","model_variant":"InternVL2-26B","model_version":"InternVL 2","name":"InternVL2-26B","open_source":true,"parameter_size":"26B","provider":"opengvlab","url":""},"internvl2-4b":{"country":"CN","id":"internvl2-4b","launched":"","modalities":["visual"],"model":"InternVL2 4B","model_family":"Intern","model_variant":"InternVL2-4B","model_version":"InternVL 2","name":"InternVL2-4B","open_source":true,"parameter_size":"4B","provider":"opengvlab","url":""},"internvl2-76b":{"country":"CN","id":"internvl2-76b","launched":"","modalities":["visual"],"model":"InternVL2 76B","model_family":"Intern","model_variant":"InternVL2-76B","model_version":"InternVL 2","name":"InternVL2-76B","open_source":true,"parameter_size":"76B","provider":"opengvlab","url":""},"internvl2-8b":{"country":"CN","id":"internvl2-8b","launched":"","modalities":["visual"],"model":"InternVL2 8B","model_family":"Intern","model_variant":"InternVL2-8B","model_version":"InternVL 2","name":"InternVL2-8B","open_source":true,"parameter_size":"8B","provider":"opengvlab","url":""},"internvl2-llama-3-76b":{"country":"CN","id":"internvl2-llama-3-76b","launched":"","modalities":["visual"],"model":"InternVL2 Llama 3 76B","model_family":"Intern","model_variant":"InternVL2-Llama-3-76B","model_version":"InternVL 2","name":"InternVL2-Llama-3-76B","open_source":true,"parameter_size":"76B","provider":"opengvlab","url":""},"internvl2.5-26b":{"country":"CN","id":"internvl2.5-26b","launched":"","modalities":["visual"],"model":"InternVL2.5 26B","model_family":"Intern","model_variant":"InternVL2.5-26B","model_version":"InternVL 2.5","name":"InternVL2.5-26B","open_source":true,"parameter_size":"26B","provider":"opengvlab","url":""},"internvl2.5-2b":{"country":"CN","id":"internvl2.5-2b","launched":"2025-12-05","modalities":["visual"],"model":"InternVL2.5 2B","model_family":"Intern","model_variant":"InternVL2.5-2B","model_version":"InternVL 2.5","name":"InternVL2.5-2B","open_source":true,"parameter_size":"2B","provider":"opengvlab","url":""},"internvl2.5-38b-rl":{"country":"CN","id":"internvl2.5-38b-rl","launched":"","modalities":["visual"],"model":"InternVL2.5 38B","model_family":"Intern","model_variant":"InternVL2.5-38B-RL","model_version":"InternVL 2.5","name":"InternVL2.5-38B-RL","open_source":true,"parameter_size":"38B","provider":"opengvlab","url":""},"internvl2.5-4b":{"country":"CN","id":"internvl2.5-4b","launched":"2024-10-21","modalities":["visual"],"model":"InternVL2.5 4B","model_family":"Intern","model_variant":"InternVL2.5-4B","model_version":"InternVL 2.5","name":"InternVL2.5-4B","open_source":true,"parameter_size":"4B","provider":"opengvlab","url":""},"internvl3":{"country":"CN","id":"internvl3","launched":"2025-04-11","modalities":["visual"],"model":"InternVL3","model_family":"Intern","model_variant":"InternVL3","model_version":"InternVL 3","name":"InternVL3","open_source":true,"parameter_size":"","provider":"opengvlab","url":""},"internvl3-2b":{"country":"CN","id":"internvl3-2b","launched":"2025-04-11","modalities":["visual"],"model":"InternVL3 2B","model_family":"Intern","model_variant":"InternVL3-2B","model_version":"InternVL 3","name":"InternVL3-2B","open_source":true,"parameter_size":"2B","provider":"opengvlab","url":""},"internvl3-78b":{"country":"CN","id":"internvl3-78b","launched":"2025-04-11","modalities":["visual"],"model":"InternVL3 78B","model_family":"Intern","model_variant":"InternVL3-78B","model_version":"InternVL 3","name":"InternVL3-78B","open_source":true,"parameter_size":"78B","provider":"opengvlab","url":""},"internvl3-8b":{"country":"CN","id":"internvl3-8b","launched":"2025-04-11","modalities":["visual"],"model":"InternVL3-8B","model_family":"Intern","model_variant":"InternVL3-8B","model_version":"InternVL 3","name":"InternVL3-8B","open_source":true,"parameter_size":"8B","provider":"opengvlab","url":""},"internvl3.5-1b":{"country":"CN","id":"internvl3.5-1b","launched":"2025-08-26","modalities":["visual"],"model":"InternVL3.5 1B","model_family":"Intern","model_variant":"InternVL3.5-1B","model_version":"InternVL 3.5","name":"InternVL3.5-1B","open_source":true,"parameter_size":"1B","provider":"opengvlab","url":""},"internvl3.5-241b":{"country":"CN","id":"internvl3.5-241b","launched":"2025-08-26","modalities":["visual"],"model":"InternVL3.5 241B","model_family":"Intern","model_variant":"InternVL3.5-241B","model_version":"InternVL 3.5","name":"InternVL3.5-241B","open_source":true,"parameter_size":"241B","provider":"opengvlab","url":""},"internvl3.5-2b":{"country":"CN","id":"internvl3.5-2b","launched":"2025-08-26","modalities":["visual"],"model":"InternVL3.5 2B","model_family":"Intern","model_variant":"InternVL3.5-2B","model_version":"InternVL 3.5","name":"InternVL3.5-2B","open_source":true,"parameter_size":"2B","provider":"opengvlab","url":""},"internvl3.5-4b":{"country":"CN","id":"internvl3.5-4b","launched":"2025-08-26","modalities":["visual"],"model":"InternVL3.5 4B","model_family":"Intern","model_variant":"InternVL3.5-4B","model_version":"InternVL 3.5","name":"InternVL3.5-4B","open_source":true,"parameter_size":"4B","provider":"opengvlab","url":""},"internvl3.5-8b":{"country":"CN","id":"internvl3.5-8b","launched":"2025-08-26","modalities":["visual"],"model":"InternVL3.5 8B","model_family":"Intern","model_variant":"InternVL3.5-8B","model_version":"InternVL 3.5","name":"InternVL3.5-8B","open_source":true,"parameter_size":"8B","provider":"opengvlab","url":""},"inversecoder-cl-13b":{"country":"US","id":"inversecoder-cl-13b","launched":"","modalities":["text"],"model":"Inversecoder CL 13B","model_family":"Inversecoder","model_variant":"Inversecoder CL 13B","model_version":"Inversecoder","name":"Inversecoder CL 13B","open_source":true,"parameter_size":"13B","provider":"inverse","url":""},"inversecoder-cl-7b":{"country":"US","id":"inversecoder-cl-7b","launched":"","modalities":["text"],"model":"Inversecoder CL 7B","model_family":"Inversecoder","model_variant":"Inversecoder CL 7B","model_version":"Inversecoder","name":"Inversecoder CL 7B","open_source":true,"parameter_size":"7B","provider":"inverse","url":""},"inversecoder-ds-6.7b":{"country":"US","id":"inversecoder-ds-6.7b","launched":"","modalities":["text"],"model":"Inversecoder DS 6.7B","model_family":"Inversecoder","model_variant":"Inversecoder DS 6.7B","model_version":"Inversecoder","name":"Inversecoder DS 6.7B","open_source":true,"parameter_size":"6.7B","provider":"inverse","url":""},"iquest-coder-v1-40b-instruct":{"country":"CN","id":"iquest-coder-v1-40b-instruct","launched":"2026-01-01","modalities":["text"],"model":"IQuest Coder V1 40B","model_family":"IQuest Coder","model_variant":"V1 40B Instruct","model_version":"IQuest Coder V1","name":"IQuest-Coder-V1-40B-Instruct","open_source":true,"parameter_size":"40B","provider":"iquest","url":""},"iquest-coder-v1-40b-loop-instruct":{"country":"CN","id":"iquest-coder-v1-40b-loop-instruct","launched":"2026-01-01","modalities":["text"],"model":"IQuest Coder V1 40B","model_family":"IQuest Coder","model_variant":"V1 40B Loop Instruct","model_version":"IQuest Coder V1","name":"IQuest-Coder-V1-40B-Loop-Instruct","open_source":true,"parameter_size":"40B","provider":"iquest","url":""},"iquest-coder-v1-40b-loop-thinking":{"country":"CN","id":"iquest-coder-v1-40b-loop-thinking","launched":"2026-01-01","modalities":["text"],"model":"IQuest Coder V1 40B","model_family":"IQuest Coder","model_variant":"V1 40B Loop Thinking","model_version":"IQuest Coder V1","name":"IQuest-Coder-V1-40B-Loop-Thinking","open_source":true,"parameter_size":"40B","provider":"iquest","url":""},"iquest-coder-v1-40b-thinking":{"country":"CN","id":"iquest-coder-v1-40b-thinking","launched":"2026-01-01","modalities":["text"],"model":"IQuest Coder V1 40B","model_family":"IQuest Coder","model_variant":"V1 40B Thinking","model_version":"IQuest Coder V1","name":"IQuest-Coder-V1-40B-Thinking","open_source":true,"parameter_size":"40B","provider":"iquest","url":""},"jamba-1.5-large":{"country":"IL","id":"jamba-1.5-large","launched":"","modalities":["text"],"model":"Jamba 1.5 Large","model_family":"Jamba","model_variant":"1.5 Large","model_version":"Jamba 1.5","name":"Jamba 1.5 Large","open_source":false,"parameter_size":"","provider":"ai21","url":""},"jamba-1.5-mini":{"country":"IL","id":"jamba-1.5-mini","launched":"","modalities":["text"],"model":"Jamba 1.5 Mini","model_family":"Jamba","model_variant":"1.5 Mini","model_version":"Jamba 1.5","name":"Jamba 1.5 Mini","open_source":false,"parameter_size":"","provider":"ai21","url":""},"jamba-1.6-large":{"country":"IL","id":"jamba-1.6-large","launched":"","modalities":["text"],"model":"Jamba 1.6 Large","model_family":"Jamba","model_variant":"Jamba 1.6 Large","model_version":"Jamba 1.6","name":"Jamba 1.6 Large","open_source":false,"parameter_size":"","provider":"ai21","url":""},"jamba-1.6-mini":{"country":"IL","id":"jamba-1.6-mini","launched":"","modalities":["text"],"model":"Jamba 1.6 Mini","model_family":"Jamba","model_variant":"Jamba 1.6 Mini","model_version":"Jamba 1.6","name":"Jamba 1.6 Mini","open_source":false,"parameter_size":"","provider":"ai21","url":""},"jamba-mini-1.6":{"country":"IL","id":"jamba-mini-1.6","launched":"2025-03-03","modalities":["text"],"model":"Jamba Mini 1.6","model_family":"Jamba","model_variant":"Mini 1.6","model_version":"Jamba Mini","name":"Jamba Mini 1.6","open_source":false,"parameter_size":"51.6B","provider":"ai21","url":""},"jamba-mini-1.7":{"country":"IL","id":"jamba-mini-1.7","launched":"2025-07-01","modalities":["text"],"model":"Jamba Mini 1.7","model_family":"Jamba","model_variant":"Mini 1.7","model_version":"Jamba Mini","name":"Jamba Mini 1.7","open_source":false,"parameter_size":"51.6B","provider":"ai21","url":""},"jamba-reasoning-3b":{"country":"IL","id":"jamba-reasoning-3b","launched":"2025-10-05","modalities":["text"],"model":"Jamba Reasoning 3B","model_family":"Jamba","model_variant":"Reasoning 3B","model_version":"Jamba Reasoning","name":"Jamba Reasoning 3B","open_source":true,"parameter_size":"3.2B","provider":"ai21","url":""},"jan-nano":{"country":"US","id":"jan-nano","launched":"2025-06-25","modalities":["text"],"model":"Jan nano","model_family":"Jan","model_variant":"nano","model_version":"Jan nano","name":"Jan nano","open_source":true,"parameter_size":"","provider":"menlo","url":""},"jan-nano-128k":{"country":"US","id":"jan-nano-128k","launched":"2025-06-25","modalities":["text"],"model":"Jan nano 128k","model_family":"Jan","model_variant":"nano-128k","model_version":"Jan nano","name":"Jan nano 128k","open_source":true,"parameter_size":"","provider":"menlo","url":""},"jan-nano-yarn":{"country":"US","id":"jan-nano-yarn","launched":"2025-06-25","modalities":["text"],"model":"Jan nano (YaRN)","model_family":"Jan","model_variant":"nano-YaRN","model_version":"Jan nano","name":"Jan nano (YaRN)","open_source":true,"parameter_size":"","provider":"menlo","url":""},"jan-v1":{"country":"US","id":"jan-v1","launched":"2025-09-10","modalities":["text"],"model":"Jan v1","model_family":"Jan","model_variant":"v1","model_version":"Jan v1","name":"Jan v1","open_source":true,"parameter_size":"4B","provider":"menlo","url":""},"jan-v1-2509":{"country":"US","id":"jan-v1-2509","launched":"","modalities":["text"],"model":"Jan v1","model_family":"Jan","model_variant":"v1-2509","model_version":"Jan v1","name":"Jan v1 2509","open_source":true,"parameter_size":"4B","provider":"menlo","url":""},"jan-v1-edge":{"country":"US","id":"jan-v1-edge","launched":"2025-09-03","modalities":["text"],"model":"Jan v1 edge","model_family":"Jan","model_variant":"v1-edge","model_version":"Jan v1","name":"Jan v1 Edge","open_source":true,"parameter_size":"1.7B","provider":"menlo","url":""},"jan-v3":{"country":"US","id":"jan-v3","launched":"","modalities":["text"],"model":"Jan v3","model_family":"Jan","model_variant":"v3","model_version":"Jan v3","name":"Jan v3","open_source":true,"parameter_size":"4B","provider":"menlo","url":""},"janus-pro":{"country":"CN","id":"janus-pro","launched":"","modalities":["visual","text"],"model":"Janus Pro","model_family":"Janus","model_variant":"Janus Pro","model_version":"Janus Pro","name":"Janus-Pro","open_source":true,"parameter_size":"","provider":"bytedance","url":""},"janus-pro-1.5b":{"country":"CN","id":"janus-pro-1.5b","launched":"","modalities":["visual","text"],"model":"Janus Pro","model_family":"Janus","model_variant":"Janus Pro 1.5B","model_version":"Janus Pro","name":"Janus Pro 1.5B","open_source":true,"parameter_size":"1.5B","provider":"bytedance","url":""},"janus-pro-7b":{"country":"CN","id":"janus-pro-7b","launched":"","modalities":["visual","text"],"model":"Janus Pro","model_family":"Janus","model_variant":"Janus Pro 7B","model_version":"Janus Pro","name":"Janus Pro 7B","open_source":true,"parameter_size":"7B","provider":"bytedance","url":""},"jina-vlm-2.4b-20251117":{"country":"DE","id":"jina-vlm-2.4b-20251117","launched":"2025-11-17","modalities":["text","visual"],"model":"jina-VLM 2.4B","model_family":"jina-VLM","model_variant":"2.4B","model_version":"jina-VLM","name":"jina-VLM 2.4B (2025-11-17)","open_source":true,"parameter_size":"2.4B","provider":"jina","url":""},"joyai-llm-flash":{"active_parameters":"3B","country":"CN","id":"joyai-llm-flash","launched":"2026-02-15","modalities":["text"],"model":"JoyAI-LLM Flash","model_family":"JoyAI-LLM","model_variant":"Flash","model_version":"JoyAI-LLM","name":"JoyAI-LLM Flash","open_source":true,"parameter_size":"48B","provider":"jd","url":"https://huggingface.co/jdopensource/JoyAI-LLM-Flash"},"joyai-llm-flash-base":{"active_parameters":"3B","country":"CN","id":"joyai-llm-flash-base","launched":"2026-02-15","modalities":["text"],"model":"JoyAI-LLM Flash","model_family":"JoyAI-LLM","model_variant":"Flash Base","model_version":"JoyAI-LLM","name":"JoyAI-LLM Flash-Base","open_source":true,"parameter_size":"48B","provider":"jd","url":"https://huggingface.co/jdopensource/JoyAI-LLM-Flash-Base"},"k-exaone":{"active_parameters":"23B","country":"KR","id":"k-exaone","launched":"2025-12-31","modalities":["text"],"model":"K-EXAONE","model_family":"K-EXAONE","model_variant":"K-EXAONE","model_version":"K-EXAONE","name":"K-EXAONE","open_source":true,"parameter_size":"236B","provider":"lg","url":""},"k2-high-70b":{"country":"AE","id":"k2-high-70b","launched":"2025-12-04","modalities":["text"],"model":"K2 V2","model_family":"K2","model_variant":"K2 V2 High","model_version":"K2 V2","name":"K2 V2 High 70B","open_source":"plus","parameter_size":"70B","provider":"mbzuai","url":""},"k2-low-70b":{"country":"AE","id":"k2-low-70b","launched":"2025-12-04","modalities":["text"],"model":"K2 V2","model_family":"K2","model_variant":"K2 V2 Low","model_version":"K2 V2","name":"K2 V2 Low 70B","open_source":"plus","parameter_size":"70B","provider":"mbzuai","url":""},"k2-medium-70b":{"country":"AE","id":"k2-medium-70b","launched":"2025-12-04","modalities":["text"],"model":"K2 V2","model_family":"K2","model_variant":"K2 V2 Medium","model_version":"K2 V2","name":"K2 V2 Medium 70B","open_source":"plus","parameter_size":"70B","provider":"mbzuai","url":""},"k2-think":{"country":"AE","id":"k2-think","launched":"2025-09-09","modalities":["text"],"model":"K2-THINK","model_family":"K2","model_variant":"K2-THINK","model_version":"K2-THINK","name":"K2-THINK","open_source":true,"parameter_size":"32B","provider":"mbzuai","url":""},"k2-v2":{"country":"AE","id":"k2-v2","launched":"2025-12-04","modalities":["text"],"model":"K2 V2","model_family":"K2","model_variant":"K2-V2","model_version":"K2 V2","name":"K2-V2","open_source":"plus","parameter_size":"70B","provider":"mbzuai","url":""},"kanana-1.5-32.5b-base":{"country":"KR","id":"kanana-1.5-32.5b-base","launched":"2025-05-23","modalities":["text"],"model":"Kanana 1.5 32.5B","model_family":"Kanana","model_variant":"1.5-32.5B-Base","model_version":"Kanana 1.5","name":"Kanana-1.5-32.5B-Base","open_source":true,"parameter_size":"32.5B","provider":"kakao","url":""},"kanana-1.5-32.5b-instruct":{"country":"KR","id":"kanana-1.5-32.5b-instruct","launched":"2025-05-23","modalities":["text"],"model":"Kanana 1.5 32.5B","model_family":"Kanana","model_variant":"1.5-32.5B-Instruct","model_version":"Kanana 1.5","name":"Kanana-1.5-32.5B-Instruct","open_source":true,"parameter_size":"32.5B","provider":"kakao","url":""},"kanana-2-30b-a3b-base-2601":{"active_parameters":"3B","country":"KR","id":"kanana-2-30b-a3b-base-2601","launched":"2026-01-15","modalities":["text"],"model":"Kanana 2 30B A3B","model_family":"Kanana","model_variant":"2-30B-A3B-Base-2601","model_version":"Kanana 2","name":"Kanana-2-30B-A3B-Base-2601","open_source":true,"parameter_size":"30B","provider":"kakao","url":""},"kanana-2-30b-a3b-instruct":{"active_parameters":"3B","country":"KR","id":"kanana-2-30b-a3b-instruct","launched":"2025-12-19","modalities":["text"],"model":"Kanana 2 30B A3B","model_family":"Kanana","model_variant":"2-30B-A3B-Instruct","model_version":"Kanana 2","name":"Kanana-2-30B-A3B-Instruct","open_source":true,"parameter_size":"30B","provider":"kakao","url":""},"kanana-2-30b-a3b-instruct-2601":{"active_parameters":"3B","country":"KR","id":"kanana-2-30b-a3b-instruct-2601","launched":"2026-01-15","modalities":["text"],"model":"Kanana 2 30B A3B","model_family":"Kanana","model_variant":"2-30B-A3B-Instruct-2601","model_version":"Kanana 2","name":"Kanana-2-30B-A3B-Instruct-2601","open_source":true,"parameter_size":"30B","provider":"kakao","url":""},"kanana-2-30b-a3b-mid-2601":{"active_parameters":"3B","country":"KR","id":"kanana-2-30b-a3b-mid-2601","launched":"2026-01-15","modalities":["text"],"model":"Kanana 2 30B A3B","model_family":"Kanana","model_variant":"2-30B-A3B-Mid-2601","model_version":"Kanana 2","name":"Kanana-2-30B-A3B-Mid-2601","open_source":true,"parameter_size":"30B","provider":"kakao","url":""},"kanana-2-30b-a3b-thinking":{"active_parameters":"3B","country":"KR","id":"kanana-2-30b-a3b-thinking","launched":"2025-12-19","modalities":["text"],"model":"Kanana 2 30B A3B","model_family":"Kanana","model_variant":"2-30B-A3B-Thinking","model_version":"Kanana 2","name":"Kanana-2-30B-A3B-Thinking","open_source":true,"parameter_size":"30B","provider":"kakao","url":""},"kanana-2-30b-a3b-thinking-2601":{"active_parameters":"3B","country":"KR","id":"kanana-2-30b-a3b-thinking-2601","launched":"2026-01-15","modalities":["text"],"model":"Kanana 2 30B A3B","model_family":"Kanana","model_variant":"2-30B-A3B-Thinking-2601","model_version":"Kanana 2","name":"Kanana-2-30B-A3B-Thinking-2601","open_source":true,"parameter_size":"30B","provider":"kakao","url":""},"kanana-nano-2.1b-instruct":{"country":"KR","id":"kanana-nano-2.1b-instruct","launched":"","modalities":["text"],"model":"Kanana Nano 2.1B","model_family":"Kanana Nano","model_variant":"Kanana Nano 2.1B Instruct","model_version":"Kanana Nano","name":"Kanana Nano 2.1B Instruct","open_source":true,"parameter_size":"2.1B","provider":"kakao","url":""},"kat-coder":{"country":"CN","id":"kat-coder","launched":"2025-10-21","modalities":["text"],"model":"KAT-Coder","model_family":"KAT","model_variant":"Coder","model_version":"KAT","name":"KAT-Coder","open_source":false,"parameter_size":"","provider":"kuaishou","url":""},"kat-coder-pro-v1":{"country":"CN","id":"kat-coder-pro-v1","launched":"","modalities":["text"],"model":"KAT-Coder Pro","model_family":"KAT","model_variant":"Coder Pro v1","model_version":"KAT-Coder Pro v1","name":"KAT-Coder Pro v1","open_source":false,"parameter_size":"","provider":"kuaishou","url":""},"kat-dev-32b":{"country":"CN","id":"kat-dev-32b","launched":"2025-10-10","modalities":["text"],"model":"KAT-Dev 32B","model_family":"KAT","model_variant":"Dev-32B","model_version":"KAT","name":"KAT-Dev-32B","open_source":true,"parameter_size":"32B","provider":"kuaishou","url":""},"kat-dev-72b-exp":{"country":"CN","id":"kat-dev-72b-exp","launched":"2025-10-10","modalities":["text"],"model":"KAT-Dev 72B","model_family":"KAT","model_variant":"Dev-72B Exp","model_version":"KAT","name":"KAT-Dev-72B-Exp","open_source":true,"parameter_size":"72B","provider":"kuaishou","url":""},"kat-v1-40b":{"country":"CN","id":"kat-v1-40b","launched":"","modalities":["text"],"model":"KAT V1 40B","model_family":"KAT","model_variant":"V1 40B","model_version":"KAT V1","name":"KAT V1 40B","open_source":false,"parameter_size":"40B","provider":"kuaishou","url":""},"keye-vl-1.5-8b":{"country":"CN","id":"keye-vl-1.5-8b","launched":"","modalities":["visual"],"model":"Keye-VL 1.5 8B","model_family":"Keye-VL","model_variant":"Keye-VL-1.5-8B","model_version":"Keye-VL 1.5","name":"Keye-VL-1.5-8B","open_source":true,"parameter_size":"8B","provider":"kuaishou","url":""},"keye-vl-8b":{"country":"CN","id":"keye-vl-8b","launched":"","modalities":["visual"],"model":"Keye-VL 8B","model_family":"Keye-VL","model_variant":"Keye-VL-8B","model_version":"Keye-VL","name":"Keye-VL 8B","open_source":true,"parameter_size":"8B","provider":"kuaishou","url":""},"kimi-dev-72b":{"country":"CN","id":"kimi-dev-72b","launched":"","modalities":["text"],"model":"Kimi Dev 72B","model_family":"Kimi","model_variant":"Kimi Dev 72B","model_version":"Kimi Dev","name":"Kimi-Dev-72B","open_source":true,"parameter_size":"72B","provider":"moonshotai","url":""},"kimi-k2":{"country":"CN","id":"kimi-k2","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"Kimi K2","model_version":"Kimi K2","name":"Kimi-K2","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-0711-preview":{"country":"CN","id":"kimi-k2-0711-preview","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"Kimi K2 0711 Preview","model_version":"Kimi K2","name":"Kimi-K2 0711 Preview","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-0905":{"country":"CN","id":"kimi-k2-0905","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"Kimi K2 0905","model_version":"Kimi K2","name":"Kimi K2 0905","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-0905-preview":{"country":"CN","id":"kimi-k2-0905-preview","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"Kimi K2 0905 Preview","model_version":"Kimi K2","name":"Kimi-K2 0905 Preview","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-base":{"country":"CN","id":"kimi-k2-base","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2 Base","model_family":"Kimi","model_variant":"Kimi K2 Base","model_version":"Kimi K2","name":"Kimi-K2 Base","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-instruct":{"country":"CN","id":"kimi-k2-instruct","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"Kimi K2","model_version":"Kimi K2","name":"Kimi K2 Instruct","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-instruct-0711":{"country":"CN","id":"kimi-k2-instruct-0711","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"Kimi K2","model_version":"Kimi K2","name":"Kimi-K2-Instruct-0711","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-instruct-0905":{"active_parameters":"32B","country":"CN","id":"kimi-k2-instruct-0905","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"Kimi K2","model_version":"Kimi K2","name":"Kimi-K2-Instruct-0905","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-instruct-0905-groq":{"active_parameters":"32B","country":"CN","id":"kimi-k2-instruct-0905-groq","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2 (Groq)","model_family":"Kimi","model_variant":"Kimi K2 Groq","model_version":"Kimi K2","name":"Kimi-K2-Instruct-0905 (Groq)","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-thinking":{"country":"CN","id":"kimi-k2-thinking","launched":"2025-11-06","modalities":["text"],"model":"Kimi K2 Thinking","model_family":"Kimi","model_variant":"Kimi K2 Thinking","model_version":"Kimi K2","name":"Kimi-K2 Thinking","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":""},"kimi-k2-turbo-preview":{"country":"CN","id":"kimi-k2-turbo-preview","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2 Turbo","model_family":"Kimi","model_variant":"Kimi K2 Turbo Preview","model_version":"Kimi K2","name":"Kimi-K2 Turbo Preview","open_source":true,"parameter_size":"","provider":"moonshotai","url":""},"kimi-k2.5":{"active_parameters":"32B","country":"CN","id":"kimi-k2.5","launched":"2026-01-27","modalities":["text","visual"],"model":"Kimi K2.5","model_family":"Kimi","model_variant":"Kimi K2.5","model_version":"Kimi K2.5","name":"Kimi-K2.5","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":"https://huggingface.co/moonshotai/Kimi-K2.5"},"kimi-k2.5-thinking":{"active_parameters":"32B","country":"CN","id":"kimi-k2.5-thinking","launched":"2026-01-27","modalities":["text","visual"],"model":"Kimi K2.5","model_family":"Kimi","model_variant":"Kimi K2.5 Thinking","model_version":"Kimi K2.5","name":"Kimi-K2.5 Thinking","open_source":true,"parameter_size":"1000B","provider":"moonshotai","url":"https://huggingface.co/moonshotai/Kimi-K2.5"},"kimi-k20895":{"country":"CN","id":"kimi-k20895","launched":"2025-07-11","modalities":["text"],"model":"Kimi K2","model_family":"Kimi","model_variant":"K20895","model_version":"Kimi K2","name":"Kimi K20895","open_source":false,"parameter_size":"","provider":"moonshot","url":""},"kimi-linear-base":{"active_parameters":"3B","country":"CN","id":"kimi-linear-base","launched":"2025-10-30","modalities":["text"],"model":"Kimi Linear","model_family":"Kimi","model_variant":"Kimi Linear Base","model_version":"Kimi Linear","name":"Kimi-Linear-Base","open_source":true,"parameter_size":"48B","provider":"moonshotai","url":""},"kimi-linear-instruct":{"active_parameters":"3B","country":"CN","id":"kimi-linear-instruct","launched":"2025-10-30","modalities":["text"],"model":"Kimi Linear","model_family":"Kimi","model_variant":"Kimi Linear Instruct","model_version":"Kimi Linear","name":"Kimi-Linear-Instruct","open_source":true,"parameter_size":"48B","provider":"moonshotai","url":""},"kimi-researcher":{"country":"CN","id":"kimi-researcher","launched":"2025-06-20","modalities":["text"],"model":"Kimi Researcher","model_family":"Kimi","model_variant":"Kimi Researcher","model_version":"Kimi Researcher","name":"Kimi Researcher","open_source":false,"parameter_size":"","provider":"moonshotai","url":""},"kimi-vl":{"country":"CN","id":"kimi-vl","launched":"2025-08-07","modalities":["visual"],"model":"Kimi VL 7B","model_family":"Kimi","model_variant":"Kimi VL","model_version":"Kimi VL","name":"Kimi VL","open_source":true,"parameter_size":"7B","provider":"moonshotai","url":""},"kimi-vl-a3b-a2b-thinking-2506":{"active_parameters":"3B","country":"CN","id":"kimi-vl-a3b-a2b-thinking-2506","launched":"2025-10-30","modalities":["visual"],"model":"Kimi VL A3B","model_family":"Kimi","model_variant":"Kimi-VL-A3B~A2B-Thinking-2506","model_version":"Kimi VL","name":"Kimi-VL-A3B~A2B-Thinking-2506","open_source":true,"parameter_size":"16B","provider":"moonshotai","url":""},"kimi-vl-a3b-thinking":{"country":"CN","id":"kimi-vl-a3b-thinking","launched":"2025-06-21","modalities":["visual"],"model":"Kimi VL A3B","model_family":"Kimi","model_variant":"Kimi VL","model_version":"Kimi VL","name":"Kimi-VL-A3B-Thinking","open_source":true,"parameter_size":"3B","provider":"moonshotai","url":""},"kimi-vl-a3b-thinking-2506":{"country":"CN","id":"kimi-vl-a3b-thinking-2506","launched":"","modalities":["visual"],"model":"Kimi VL A3B","model_family":"Kimi","model_variant":"Kimi-VL-A3B-Thinking-2506","model_version":"Kimi VL","name":"Kimi VL A3B Thinking 2506","open_source":true,"parameter_size":"16.4B","provider":"moonshotai","url":""},"ko-r1-7b-v2.1":{"country":"KR","id":"ko-r1-7b-v2.1","launched":"","modalities":["text"],"model":"Ko-R1-7B-v2.1","model_family":"Ko-R1","model_variant":"Ko-R1-7B-v2.1","model_version":"Ko-R1","name":"Ko-R1-7B-v2.1","open_source":true,"parameter_size":"7B","provider":"onelineai","url":""},"koala-13b":{"country":"US","id":"koala-13b","launched":"","modalities":["text"],"model":"Koala 13B","model_family":"Koala","model_variant":"Koala 13B","model_version":"Koala","name":"Koala 13B","open_source":true,"parameter_size":"13B","provider":"ucberkeley","url":""},"kwai-coder-23b-a4b-v1":{"active_parameters":"4B","country":"CN","id":"kwai-coder-23b-a4b-v1","launched":"","modalities":["text"],"model":"KwaiCoder 23B","model_family":"KwaiCoder","model_variant":"23B A4B v1","model_version":"KwaiCoder","name":"KwaiCoder 23B A4B v1","open_source":true,"parameter_size":"23B","provider":"kuaishou","url":""},"lfm2-1.2b":{"country":"US","id":"lfm2-1.2b","launched":"2025-07-10","modalities":["text"],"model":"LFM2 1.2B","model_family":"LFM","model_variant":"LFM2-1.2B","model_version":"LFM2","name":"LFM2 1.2B","open_source":true,"parameter_size":"1.2B","provider":"liquidai","url":""},"lfm2-2.6b":{"country":"US","id":"lfm2-2.6b","launched":"2025-07-10","modalities":["text"],"model":"LFM2 2.6B","model_family":"LFM","model_variant":"LFM2-2.6B","model_version":"LFM2","name":"LFM2 2.6B","open_source":true,"parameter_size":"2.6B","provider":"liquidai","url":""},"lfm2-2.6b-exp":{"country":"US","id":"lfm2-2.6b-exp","launched":"2025-12-29","modalities":["text"],"model":"LFM2 2.6B Exp","model_family":"LFM","model_variant":"LFM2-2.6B-Exp","model_version":"LFM2","name":"LFM2 2.6B Exp","open_source":true,"parameter_size":"2.6B","provider":"liquidai","url":""},"lfm2-350m":{"country":"US","id":"lfm2-350m","launched":"2025-07-10","modalities":["text"],"model":"LFM2 350M","model_family":"LFM","model_variant":"LFM2-350M","model_version":"LFM2","name":"LFM2 350M","open_source":true,"parameter_size":"0.35B","provider":"liquidai","url":""},"lfm2-350m-math":{"country":"US","id":"lfm2-350m-math","launched":"2025-08-25","modalities":["text"],"model":"LFM2 350M Math","model_family":"LFM","model_variant":"LFM2-350M-Math","model_version":"LFM2","name":"LFM2 350M Math","open_source":true,"parameter_size":"0.35B","provider":"liquidai","url":""},"lfm2-700m":{"country":"US","id":"lfm2-700m","launched":"2025-07-10","modalities":["text"],"model":"LFM2 700M","model_family":"LFM","model_variant":"LFM2-700M","model_version":"LFM2","name":"LFM2 700M","open_source":true,"parameter_size":"0.7B","provider":"liquidai","url":""},"lfm2-8b-a1b":{"country":"US","id":"lfm2-8b-a1b","launched":"2025-07-10","modalities":["text"],"model":"LFM2 8B A1B","model_family":"LFM","model_variant":"LFM2-8B-A1B","model_version":"LFM2","name":"LFM2 8B A1B","open_source":true,"parameter_size":"8B","provider":"liquidai","url":""},"lfm2-audio-1.5b":{"country":"US","id":"lfm2-audio-1.5b","launched":"2025-10-22","modalities":["text","audio"],"model":"LFM2-Audio 1.5B","model_family":"LFM","model_variant":"LFM2-Audio-1.5B","model_version":"LFM2-Audio","name":"LFM2-Audio-1.5B","open_source":true,"parameter_size":"1.5B","provider":"liquidai","url":""},"lfm2-vl-1.6b":{"country":"US","id":"lfm2-vl-1.6b","launched":"2025-10-22","modalities":["text","visual"],"model":"LFM2-VL 1.6B","model_family":"LFM","model_variant":"LFM2-VL-1.6B","model_version":"LFM2-VL","name":"LFM2-VL-1.6B","open_source":true,"parameter_size":"1.6B","provider":"liquidai","url":""},"lfm2-vl-3b":{"country":"US","id":"lfm2-vl-3b","launched":"2025-10-22","modalities":["text","visual"],"model":"LFM2-VL 3B","model_family":"LFM","model_variant":"LFM2-VL-3B","model_version":"LFM2-VL","name":"LFM2-VL-3B","open_source":true,"parameter_size":"3B","provider":"liquidai","url":""},"lfm2-vl-450m":{"country":"US","id":"lfm2-vl-450m","launched":"2025-10-22","modalities":["text","visual"],"model":"LFM2-VL 450M","model_family":"LFM","model_variant":"LFM2-VL-450M","model_version":"LFM2-VL","name":"LFM2-VL-450M","open_source":true,"parameter_size":"0.45B","provider":"liquidai","url":""},"lfm2.5-1.2b-instruct":{"country":"US","id":"lfm2.5-1.2b-instruct","launched":"2026-01-06","modalities":["text"],"model":"LFM2.5 1.2B Instruct","model_family":"LFM","model_variant":"LFM2.5-1.2B Instruct","model_version":"LFM2.5","name":"LFM2.5 1.2B Instruct","open_source":true,"parameter_size":"1.2B","provider":"liquidai","url":""},"lfm2.5-1.2b-thinking":{"country":"US","id":"lfm2.5-1.2b-thinking","launched":"2026-01-20","modalities":["text"],"model":"LFM2.5 1.2B","model_family":"LFM","model_variant":"LFM2.5-1.2B-Thinking","model_version":"LFM2.5","name":"LFM2.5-1.2B-Thinking","open_source":true,"parameter_size":"1.2B","provider":"liquidai","url":""},"lfm2.5-audio-1.5b":{"country":"US","id":"lfm2.5-audio-1.5b","launched":"2026-01-06","modalities":["text","audio"],"model":"LFM2.5-Audio 1.5B","model_family":"LFM","model_variant":"LFM2.5-Audio-1.5B","model_version":"LFM2.5-Audio","name":"LFM2.5-Audio-1.5B","open_source":true,"parameter_size":"1.5B","provider":"liquidai","url":""},"lfm2.5-vl-1.6b":{"country":"US","id":"lfm2.5-vl-1.6b","launched":"2026-01-06","modalities":["text","visual"],"model":"LFM2.5-VL 1.6B","model_family":"LFM","model_variant":"LFM2.5-VL-1.6B","model_version":"LFM2.5-VL","name":"LFM2.5-VL-1.6B","open_source":true,"parameter_size":"1.6B","provider":"liquidai","url":""},"lg-exaone-32b":{"country":"KR","id":"lg-exaone-32b","launched":"2025-07-07","modalities":["text"],"model":"LG EXAONE 32B","model_family":"EXAONE","model_variant":"LG EXAONE 32B","model_version":"EXAONE 4.0","name":"LG EXAONE 32B","open_source":true,"parameter_size":"32B","provider":"lg","url":""},"ling-1t":{"active_parameters":"50B","country":"CN","id":"ling-1t","launched":"2025-10-03","modalities":["text"],"model":"Ling 1T","model_family":"Ling","model_variant":"1T","model_version":"Ling 1T","name":"Ling 1T","open_source":true,"parameter_size":"1000B","provider":"antgroup","url":""},"ling-2.5-1t":{"active_parameters":"63B","country":"CN","id":"ling-2.5-1t","launched":"2026-02-16","modalities":["text"],"model":"Ling 2.5 1T","model_family":"Ling","model_variant":"2.5 1T","model_version":"Ling 2.5","name":"Ling 2.5 1T","open_source":true,"parameter_size":"1000B","provider":"antgroup","url":"https://huggingface.co/inclusionAI/Ling-2.5-1T"},"ling-flash-2.0":{"active_parameters":"6.1B","country":"CN","id":"ling-flash-2.0","launched":"2025-09-17","modalities":["text"],"model":"Ling Flash 2.0","model_family":"Ling","model_variant":"Flash 2.0","model_version":"Ling 2.0","name":"Ling Flash 2.0","open_source":true,"parameter_size":"100B","provider":"antgroup","url":""},"ling-mini-2.0":{"active_parameters":"1.4B","country":"CN","id":"ling-mini-2.0","launched":"2025-09-08","modalities":["text"],"model":"Ling mini 2.0","model_family":"Ling","model_variant":"mini-2.0","model_version":"Ling 2.0","name":"Ling mini 2.0","open_source":true,"parameter_size":"16B","provider":"antgroup","url":"https://huggingface.co/inclusionAI/Ling-mini-2.0"},"llada-2.0":{"country":"CN","id":"llada-2.0","launched":"2025-12-11","modalities":["text"],"model":"LLaDA","model_family":"LLaDA","model_variant":"LLaDA 2.0","model_version":"LLaDA 2.0","name":"LLaDA 2.0","open_source":true,"parameter_size":"","provider":"antgroup","url":""},"llada-8b-instruct":{"country":"CN","id":"llada-8b-instruct","launched":"2025-02-14","modalities":["text"],"model":"LLaDA 8B","model_family":"LLaDA","model_variant":"LLaDA 8B Instruct","model_version":"LLaDA","name":"LLaDA-8B-Instruct","open_source":true,"parameter_size":"8B","provider":"antgroup","url":""},"llada2.0-flash":{"active_parameters":"6B","country":"CN","id":"llada2.0-flash","launched":"2025-12-11","modalities":["text"],"model":"LLaDA 2.0 Flash","model_family":"LLaDA","model_variant":"LLaDA2.0 Flash","model_version":"LLaDA 2.0","name":"LLaDA2.0 Flash","open_source":true,"parameter_size":"100B","provider":"antgroup","url":""},"llada2.0-flash-preview":{"active_parameters":"6B","country":"CN","id":"llada2.0-flash-preview","launched":"2025-12-11","modalities":["text"],"model":"LLaDA 2.0 Flash","model_family":"LLaDA","model_variant":"LLaDA2.0 Flash Preview","model_version":"LLaDA 2.0","name":"LLaDA2.0 Flash Preview","open_source":true,"parameter_size":"100B","provider":"antgroup","url":""},"llada2.0-mini":{"country":"CN","id":"llada2.0-mini","launched":"2025-12-11","modalities":["text"],"model":"LLaDA 2.0 mini","model_family":"LLaDA","model_variant":"LLaDA2.0 mini","model_version":"LLaDA 2.0","name":"LLaDA2.0-mini","open_source":true,"parameter_size":"","provider":"antgroup","url":""},"llada2.1-flash":{"country":"CN","id":"llada2.1-flash","launched":"2026-02-10","modalities":["text"],"model":"LLaDA 2.1 Flash","model_family":"LLaDA","model_variant":"LLaDA2.1 Flash","model_version":"LLaDA 2.1","name":"LLaDA2.1-Flash","open_source":true,"parameter_size":"103B","provider":"antgroup","url":""},"llada2.1-flash-q":{"country":"CN","id":"llada2.1-flash-q","launched":"2026-02-10","modalities":["text"],"model":"LLaDA 2.1 Flash","model_family":"LLaDA","model_variant":"LLaDA2.1 Flash Q","model_version":"LLaDA 2.1","name":"LLaDA2.1-Flash (Q Mode)","open_source":true,"parameter_size":"103B","provider":"antgroup","url":""},"llada2.1-flash-s":{"country":"CN","id":"llada2.1-flash-s","launched":"2026-02-10","modalities":["text"],"model":"LLaDA 2.1 Flash","model_family":"LLaDA","model_variant":"LLaDA2.1 Flash S","model_version":"LLaDA 2.1","name":"LLaDA2.1-Flash (S Mode)","open_source":true,"parameter_size":"103B","provider":"antgroup","url":""},"llada2.1-mini":{"country":"CN","id":"llada2.1-mini","launched":"2026-02-10","modalities":["text"],"model":"LLaDA 2.1 mini","model_family":"LLaDA","model_variant":"LLaDA2.1 mini","model_version":"LLaDA 2.1","name":"LLaDA2.1-mini","open_source":true,"parameter_size":"16B","provider":"antgroup","url":""},"llada2.1-mini-q":{"country":"CN","id":"llada2.1-mini-q","launched":"2026-02-10","modalities":["text"],"model":"LLaDA 2.1 mini","model_family":"LLaDA","model_variant":"LLaDA2.1 mini Q","model_version":"LLaDA 2.1","name":"LLaDA2.1-mini (Q Mode)","open_source":true,"parameter_size":"16B","provider":"antgroup","url":""},"llada2.1-mini-s":{"country":"CN","id":"llada2.1-mini-s","launched":"2026-02-10","modalities":["text"],"model":"LLaDA 2.1 mini","model_family":"LLaDA","model_variant":"LLaDA2.1 mini S","model_version":"LLaDA 2.1","name":"LLaDA2.1-mini (S Mode)","open_source":true,"parameter_size":"16B","provider":"antgroup","url":""},"llama-1-13b":{"country":"US","id":"llama-1-13b","launched":"2023-02-24","modalities":["text"],"model":"Llama 1 13B","model_family":"Llama","model_variant":"Llama-13B","model_version":"Llama 1","name":"Llama 1 13B","open_source":true,"parameter_size":"13B","provider":"meta","url":""},"llama-1-33b":{"country":"US","id":"llama-1-33b","launched":"2023-02-24","modalities":["text"],"model":"Llama 1 33B","model_family":"Llama","model_variant":"Llama-33B","model_version":"Llama 1","name":"Llama-1 33B","open_source":true,"parameter_size":"33B","provider":"meta","url":""},"llama-1-65b":{"country":"US","id":"llama-1-65b","launched":"2023-02-24","modalities":["text"],"model":"Llama 1 65B","model_family":"Llama","model_variant":"Llama-65B","model_version":"Llama 1","name":"Llama-1 65B","open_source":true,"parameter_size":"65B","provider":"meta","url":""},"llama-1-7b":{"country":"US","id":"llama-1-7b","launched":"2023-02-24","modalities":["text"],"model":"Llama 1 7B","model_family":"Llama","model_variant":"Llama-7B","model_version":"Llama 1","name":"Llama 1 7B","open_source":true,"parameter_size":"7B","provider":"meta","url":""},"llama-1b":{"country":"US","id":"llama-1b","launched":"2023-02-01","modalities":["text"],"model":"Llama 1 1B","model_family":"Llama","model_variant":"Llama-1B-Instruct","model_version":"Llama 1","name":"Llama 1 1B","open_source":true,"parameter_size":"1B","provider":"meta","url":""},"llama-1b-pretrained":{"country":"US","id":"llama-1b-pretrained","launched":"2023-02-01","modalities":["text"],"model":"Llama 1 1B","model_family":"Llama","model_variant":"Llama-1B-Pretrained","model_version":"Llama 1","name":"Llama 1B Pretrained","open_source":true,"parameter_size":"1B","provider":"meta","url":""},"llama-2-13b":{"country":"US","id":"llama-2-13b","launched":"2023-06-18","modalities":["text"],"model":"Llama 2 13B","model_family":"Llama","model_variant":"Llama 2 13B","model_version":"Llama 2","name":"Llama 2 13B","open_source":true,"parameter_size":"13B","provider":"meta","url":""},"llama-2-13b-chat":{"country":"US","id":"llama-2-13b-chat","launched":"2023-06-18","modalities":["text"],"model":"Llama 2 13B","model_family":"Llama","model_variant":"Llama 2 13B","model_version":"Llama 2","name":"Llama 2 13B Chat","open_source":true,"parameter_size":"13B","provider":"meta","url":""},"llama-2-70b":{"country":"US","id":"llama-2-70b","launched":"2023-07-23","modalities":["text"],"model":"Llama 2 70B","model_family":"Llama","model_variant":"Llama 2 70B","model_version":"Llama 2","name":"Llama 2 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-2-70b-chat":{"country":"US","id":"llama-2-70b-chat","launched":"2023-07-23","modalities":["text"],"model":"Llama 2 70B","model_family":"Llama","model_variant":"Llama 2 70B","model_version":"Llama 2","name":"Llama 2 70B Chat","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-2-7b":{"country":"US","id":"llama-2-7b","launched":"2023-06-18","modalities":["text"],"model":"Llama 2 7B","model_family":"Llama","model_variant":"Llama 2 7B","model_version":"Llama 2","name":"Llama 2 7B","open_source":true,"parameter_size":"7B","provider":"meta","url":""},"llama-2-7b-chat":{"country":"US","id":"llama-2-7b-chat","launched":"2023-06-18","modalities":["text"],"model":"Llama 2 7B","model_family":"Llama","model_variant":"Llama 2 7B","model_version":"Llama 2","name":"Llama 2 7B Chat","open_source":true,"parameter_size":"7B","provider":"meta","url":""},"llama-3-70b":{"country":"US","id":"llama-3-70b","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 70B","model_family":"Llama","model_variant":"Llama3-70B","model_version":"Llama 3","name":"Llama 3 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3-70b-base":{"country":"US","id":"llama-3-70b-base","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 70B","model_family":"Llama","model_variant":"Llama3-70B","model_version":"Llama 3","name":"Llama 3 70B Base","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3-70b-instruct":{"country":"US","id":"llama-3-70b-instruct","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 70B","model_family":"Llama","model_variant":"Llama 3 70B","model_version":"Llama 3","name":"Llama 3 70B Instruct","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3-70b-synthia-v3.5":{"country":"US","id":"llama-3-70b-synthia-v3.5","launched":"","modalities":["text"],"model":"Llama 3.70B Synthia V3.5","model_family":"Llama","model_variant":"Llama 3.70B Synthia V3.5","model_version":"Llama 1","name":"Llama 3.70B Synthia V3.5","open_source":true,"parameter_size":"3.70B","provider":"meta","url":""},"llama-3-8b":{"country":"US","id":"llama-3-8b","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 8B","model_family":"Llama","model_variant":"Llama3-8B","model_version":"Llama 3","name":"Llama 3 8B","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama-3-8b-base":{"country":"US","id":"llama-3-8b-base","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 8B","model_family":"Llama","model_variant":"Llama3-8B","model_version":"Llama 3","name":"Llama 3 8B Base","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama-3-8b-instruct":{"country":"US","id":"llama-3-8b-instruct","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 8B","model_family":"Llama","model_variant":"Llama 3 8B","model_version":"Llama 3","name":"Llama 3 8B Instruct","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama-3.1-3b":{"country":"US","id":"llama-3.1-3b","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 3B","model_family":"Llama","model_variant":"Llama3.1-3B","model_version":"Llama 3.1","name":"Llama 3.1 3B","open_source":true,"parameter_size":"3B","provider":"meta","url":""},"llama-3.1-405b":{"country":"US","id":"llama-3.1-405b","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 405B","model_family":"Llama","model_variant":"Llama3.1-405B","model_version":"Llama 3.1","name":"Llama 3.1 405B","open_source":true,"parameter_size":"405B","provider":"meta","url":""},"llama-3.1-405b-base":{"country":"US","id":"llama-3.1-405b-base","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 405B","model_family":"Llama","model_variant":"Llama3.1-405B","model_version":"Llama 3.1","name":"Llama 3.1 405B Base","open_source":true,"parameter_size":"405B","provider":"meta","url":""},"llama-3.1-405b-instruct":{"country":"US","id":"llama-3.1-405b-instruct","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 405B","model_family":"Llama","model_variant":"Llama 3.1 405B Instruct","model_version":"Llama 3.1","name":"Llama 3.1 405B Instruct","open_source":true,"parameter_size":"405B","provider":"meta","url":""},"llama-3.1-405b-instruct-bf16":{"country":"US","id":"llama-3.1-405b-instruct-bf16","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 405B","model_family":"Llama","model_variant":"Llama 3.1 405B BF16","model_version":"Llama 3.1","name":"Llama 3.1 405B Instruct BF16","open_source":true,"parameter_size":"405B","provider":"meta","url":""},"llama-3.1-405b-instruct-fp8":{"country":"US","id":"llama-3.1-405b-instruct-fp8","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 405B","model_family":"Llama","model_variant":"Llama 3.1 405B FP8","model_version":"Llama 3.1","name":"Llama 3.1 405B Instruct FP8","open_source":true,"parameter_size":"405B","provider":"meta","url":""},"llama-3.1-70b":{"country":"US","id":"llama-3.1-70b","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 70B","model_family":"Llama","model_variant":"Llama3.1-70B","model_version":"Llama 3.1","name":"Llama 3.1 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3.1-70b-base":{"country":"US","id":"llama-3.1-70b-base","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 70B","model_family":"Llama","model_variant":"Llama3.1-70B","model_version":"Llama 3.1","name":"Llama 3.1 70B Base","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3.1-70b-instruct":{"country":"US","id":"llama-3.1-70b-instruct","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 70B","model_family":"Llama","model_variant":"Llama 3.1 70B Instruct","model_version":"Llama 3.1","name":"Llama 3.1 70B Instruct","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3.1-8b":{"country":"US","id":"llama-3.1-8b","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 8B","model_family":"Llama","model_variant":"Llama3.1-8B","model_version":"Llama 3.1","name":"Llama 3.1 8B","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama-3.1-8b-base":{"country":"US","id":"llama-3.1-8b-base","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 8B","model_family":"Llama","model_variant":"Llama3.1-8B","model_version":"Llama 3.1","name":"Llama 3.1 8B Base","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama-3.1-8b-instruct":{"country":"US","id":"llama-3.1-8b-instruct","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 8B","model_family":"Llama","model_variant":"Llama3.1-8B Instruct","model_version":"Llama 3.1","name":"Llama 3.1 8B Instruct","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama-3.1-nemotron-51b-instruct":{"country":"US","id":"llama-3.1-nemotron-51b-instruct","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Nemotron 51B","model_family":"Llama","model_variant":"Llama 3.1 Nemotron 51B","model_version":"Llama 3.1","name":"Llama 3.1 Nemotron 51B Instruct","open_source":true,"parameter_size":"51B","provider":"nvidia","url":""},"llama-3.1-nemotron-70b":{"country":"US","id":"llama-3.1-nemotron-70b","launched":"2024-10-01","modalities":["text"],"model":"Llama 3.1 Nemotron 70B","model_family":"Nemotron","model_variant":"Llama3.1-Nemotron-70B","model_version":"Llama 3.1 Nemotron","name":"Llama 3.1 Nemotron 70B","open_source":true,"parameter_size":"70B","provider":"nvidia","url":""},"llama-3.1-nemotron-70b-instruct":{"country":"US","id":"llama-3.1-nemotron-70b-instruct","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Nemotron 70B","model_family":"Llama","model_variant":"Llama 3.1 Nemotron 70B Instruct","model_version":"Llama 3.1","name":"Llama 3.1 Nemotron 70B Instruct","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3.1-nemotron-70b-instruct-hf":{"country":"US","id":"llama-3.1-nemotron-70b-instruct-hf","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Nemotron 70B Instruct HF","model_family":"Llama","model_variant":"Llama 3.1 Nemotron 70B Instruct HF","model_version":"Llama 3.1","name":"Llama 3.1 Nemotron 70B Instruct HF","open_source":true,"parameter_size":"70B","provider":"nvidia","url":""},"llama-3.1-nemotron-nano-8b-v1":{"country":"US","id":"llama-3.1-nemotron-nano-8b-v1","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Nemotron Nano","model_family":"Llama","model_variant":"Llama 3.1 Nemotron Nano 8B","model_version":"Llama 3.1","name":"Llama 3.1 Nemotron Nano 8B v1","open_source":true,"parameter_size":"8B","provider":"nvidia","published_date":"2025-03-16","url":""},"llama-3.1-nemotron-ultra-253b-v1":{"country":"US","id":"llama-3.1-nemotron-ultra-253b-v1","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Nemotron Ultra","model_family":"Llama","model_variant":"Llama 3.1 Nemotron Ultra 253B","model_version":"Llama 3.1","name":"Llama 3.1 Nemotron Ultra 253B v1","open_source":true,"parameter_size":"253B","provider":"nvidia","url":""},"llama-3.1-tulu-3-405b":{"country":"US","id":"llama-3.1-tulu-3-405b","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 405B","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 405B","open_source":true,"parameter_size":"405B","provider":"allenai","url":""},"llama-3.1-tulu-3-405b-dpo":{"country":"US","id":"llama-3.1-tulu-3-405b-dpo","launched":"2024-11-20","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 405B DPO","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 405B DPO","open_source":"plus","parameter_size":"405B","provider":"allenai","url":""},"llama-3.1-tulu-3-405b-sft":{"country":"US","id":"llama-3.1-tulu-3-405b-sft","launched":"2024-11-20","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 405B SFT","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 405B SFT","open_source":"plus","parameter_size":"405B","provider":"allenai","url":""},"llama-3.1-tulu-3-70b":{"country":"US","id":"llama-3.1-tulu-3-70b","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 70B","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 70B","open_source":true,"parameter_size":"70B","provider":"allenai","url":""},"llama-3.1-tulu-3-70b-dpo":{"country":"US","id":"llama-3.1-tulu-3-70b-dpo","launched":"2024-11-20","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 70B DPO","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 70B DPO","open_source":"plus","parameter_size":"70B","provider":"allenai","url":""},"llama-3.1-tulu-3-70b-sft":{"country":"US","id":"llama-3.1-tulu-3-70b-sft","launched":"2024-11-20","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 70B SFT","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 70B SFT","open_source":"plus","parameter_size":"70B","provider":"allenai","url":""},"llama-3.1-tulu-3-8b":{"country":"US","id":"llama-3.1-tulu-3-8b","launched":"2024-06-23","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 8B","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 8B","open_source":true,"parameter_size":"8B","provider":"allenai","url":""},"llama-3.1-tulu-3-8b-dpo":{"country":"US","id":"llama-3.1-tulu-3-8b-dpo","launched":"2024-11-20","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 8B DPO","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 8B DPO","open_source":"plus","parameter_size":"8B","provider":"allenai","url":""},"llama-3.1-tulu-3-8b-sft":{"country":"US","id":"llama-3.1-tulu-3-8b-sft","launched":"2024-11-20","modalities":["text"],"model":"Llama 3.1 Tulu 3","model_family":"Llama","model_variant":"Llama 3.1 Tulu 3 8B SFT","model_version":"Tulu 3","name":"Llama 3.1 Tulu 3 8B SFT","open_source":"plus","parameter_size":"8B","provider":"allenai","url":""},"llama-3.2-1.24b":{"country":"US","id":"llama-3.2-1.24b","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 1.24B","model_family":"Llama","model_variant":"Llama3.2-1.24B-Instruct","model_version":"Llama 3.2","name":"Llama 3.2 1.24B","open_source":true,"parameter_size":"1.24B","provider":"meta","url":""},"llama-3.2-1.24b-base":{"country":"US","id":"llama-3.2-1.24b-base","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 1.24B","model_family":"Llama","model_variant":"Llama3.2-1.24B-Base","model_version":"Llama 3.2","name":"Llama 3.2 1.24B Base","open_source":true,"parameter_size":"1.24B","provider":"meta","url":""},"llama-3.2-11b-instruct":{"country":"US","id":"llama-3.2-11b-instruct","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 11B","model_family":"Llama","model_variant":"Llama3.2-11B Instruct","model_version":"Llama 3.2","name":"Llama 3.2 11B Instruct","open_source":true,"parameter_size":"11B","provider":"meta","url":""},"llama-3.2-1b":{"country":"US","id":"llama-3.2-1b","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 1B","model_family":"Llama","model_variant":"Llama3.2-1B","model_version":"Llama 3.2","name":"Llama 3.2 1B","open_source":true,"parameter_size":"1B","provider":"meta","url":""},"llama-3.2-1b-instruct":{"country":"US","id":"llama-3.2-1b-instruct","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 1B","model_family":"Llama","model_variant":"Llama3.2-1B Instruct","model_version":"Llama 3.2","name":"Llama 3.2 1B Instruct","open_source":true,"parameter_size":"1B","provider":"meta","url":""},"llama-3.2-3b":{"country":"US","id":"llama-3.2-3b","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 3B","model_family":"Llama","model_variant":"Llama3.2-3B","model_version":"Llama 3.2","name":"Llama 3.2 3B","open_source":true,"parameter_size":"3B","provider":"meta","url":""},"llama-3.2-3b-base":{"country":"US","id":"llama-3.2-3b-base","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 3B","model_family":"Llama","model_variant":"Llama3.2-3B-Base","model_version":"Llama 3.2","name":"Llama 3.2 3B Base","open_source":true,"parameter_size":"3B","provider":"meta","url":""},"llama-3.2-3b-instruct":{"country":"US","id":"llama-3.2-3b-instruct","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 3B","model_family":"Llama","model_variant":"Llama3.2-3B Instruct","model_version":"Llama 3.2","name":"Llama 3.2 3B Instruct","open_source":true,"parameter_size":"3B","provider":"meta","url":""},"llama-3.2-90b-instruct":{"country":"US","id":"llama-3.2-90b-instruct","launched":"2024-09-24","modalities":["text"],"model":"Llama 3.2 90B","model_family":"Llama","model_variant":"Llama3.2-90B Instruct","model_version":"Llama 3.2","name":"Llama 3.2 90B Instruct","open_source":true,"parameter_size":"90B","provider":"meta","url":""},"llama-3.2-90b-vision":{"country":"US","id":"llama-3.2-90b-vision","launched":"2024-09-24","modalities":["visual","text"],"model":"Llama 3.2V 90B","model_family":"Llama","model_variant":"Llama3.2-90B","model_version":"Llama 3.2","name":"Llama-3.2-90B-Vision-Instruct","open_source":true,"parameter_size":"90B","provider":"meta","url":""},"llama-3.2-vision-11b-instruct":{"country":"US","id":"llama-3.2-vision-11b-instruct","launched":"2024-09-24","modalities":["visual","text"],"model":"Llama 3.2V 11B","model_family":"Llama","model_variant":"Llama3.2 Vision 11B Instruct","model_version":"Llama 3.2","name":"Llama 3.2 Vision 11B Instruct","open_source":true,"parameter_size":"11B","provider":"meta","url":""},"llama-3.2-vision-90b-instruct":{"country":"US","id":"llama-3.2-vision-90b-instruct","launched":"2024-09-24","modalities":["visual","text"],"model":"Llama 3.2V 90B","model_family":"Llama","model_variant":"Llama3.2 Vision 90B Instruct","model_version":"Llama 3.2","name":"Llama 3.2 Vision 90B Instruct","open_source":true,"parameter_size":"90B","provider":"meta","url":""},"llama-3.2.1b-instruct":{"country":"US","id":"llama-3.2.1b-instruct","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 2.1B","model_family":"Llama","model_variant":"Llama 3 2.1B Instruct","model_version":"Llama 3","name":"Llama 3 2.1B Instruct","open_source":true,"parameter_size":"2.1B","provider":"meta","url":""},"llama-3.2v-11b-instruct":{"country":"US","id":"llama-3.2v-11b-instruct","launched":"2024-09-24","modalities":["text","visual"],"model":"Llama 3.2V 11B","model_family":"Llama","model_variant":"Llama3.2V-11B-Instruct","model_version":"Llama 3.2","name":"Llama-3.2V-11B-Instruct","open_source":true,"parameter_size":"11B","provider":"meta","url":""},"llama-3.2v-90b-instruct":{"country":"US","id":"llama-3.2v-90b-instruct","launched":"2024-09-24","modalities":["text","visual"],"model":"Llama 3.2V 90B","model_family":"Llama","model_variant":"Llama3.2V-90B-Instruct","model_version":"Llama 3.2","name":"Llama-3.2V-90B-Instruct","open_source":true,"parameter_size":"90B","provider":"meta","url":""},"llama-3.3-70b":{"country":"US","id":"llama-3.3-70b","launched":"2024-12-07","modalities":["text"],"model":"Llama 3.3 70B","model_family":"Llama","model_variant":"Llama3.3-70B","model_version":"Llama 3.3","name":"Llama 3.3 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3.3-70b-instruct":{"country":"US","id":"llama-3.3-70b-instruct","launched":"2024-12-07","modalities":["text"],"model":"Llama 3.3 70B","model_family":"Llama","model_variant":"Llama3.3-70B Instruct","model_version":"Llama 3.3","name":"Llama 3.3 70B Instruct","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3.3-instruct-turbo-70b":{"country":"US","id":"llama-3.3-instruct-turbo-70b","launched":"2024-12-07","modalities":["text"],"model":"Llama 3.3 70B","model_family":"Llama","model_variant":"Llama3.3-70B Instruct Turbo","model_version":"Llama 3.3","name":"Llama 3.3 Instruct Turbo 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-3.3-nemotron-49b-super-v1":{"country":"US","id":"llama-3.3-nemotron-49b-super-v1","launched":"2024-12-07","modalities":["text"],"model":"Llama 3.3 Nemotron","model_family":"Llama","model_variant":"Llama 3.3 Nemotron 49B","model_version":"Llama 3.3","name":"Llama 3.3 Nemotron 49B Super v1","open_source":true,"parameter_size":"49B","provider":"nvidia","url":""},"llama-3.3-nemotron-super-4":{"country":"US","id":"llama-3.3-nemotron-super-4","launched":"","modalities":["text"],"model":"Llama-3.3 Nemotron Super 4","model_family":"Nemotron","model_variant":"Nemotron Super","model_version":"Nemotron Super","name":"Llama-3.3 Nemotron Super 4","open_source":true,"parameter_size":"49B","provider":"nvidia","url":""},"llama-3.3-nemotron-super-thinking":{"country":"US","id":"llama-3.3-nemotron-super-thinking","launched":"","modalities":["text"],"model":"Llama-3.3 Nemotron Super","model_family":"Nemotron","model_variant":"Nemotron Super Thinking","model_version":"Nemotron Super","name":"Llama-3.3 Nemotron Super (Thinking)","open_source":true,"parameter_size":"49B","provider":"nvidia","url":""},"llama-3.70b-instruct":{"country":"US","id":"llama-3.70b-instruct","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 70B","model_family":"Llama","model_variant":"Llama 3.70B Instruct","model_version":"Llama 3","name":"Llama 3.70B Instruct","open_source":true,"parameter_size":"3.70B","provider":"meta","url":""},"llama-3.70b-synthia-v3.5":{"country":"US","id":"llama-3.70b-synthia-v3.5","launched":"","modalities":["text"],"model":"Llama 3 70B Synthia V3.5","model_family":"Llama","model_variant":"Llama 3.70B Synthia V3.5","model_version":"Llama 1","name":"Llama 3.70B Synthia V3.5","open_source":true,"parameter_size":"3.70B","provider":"meta","url":""},"llama-3.8b-instruct":{"country":"US","id":"llama-3.8b-instruct","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 8B","model_family":"Llama","model_variant":"Llama 3.8B Instruct","model_version":"Llama 3","name":"Llama 3.8B Instruct","open_source":true,"parameter_size":"3.8B","provider":"meta","url":""},"llama-33b-oasst-rlhf":{"country":"US","id":"llama-33b-oasst-rlhf","launched":"","modalities":["text"],"model":"LLaMA 33B OASST RLHF","model_family":"LLaMA","model_variant":"33B OASST RLHF","model_version":"LLaMA OASST","name":"LLaMA 33B OASST RLHF","open_source":true,"parameter_size":"33B","provider":"openassistant","url":""},"llama-33b-oasst-sft":{"country":"US","id":"llama-33b-oasst-sft","launched":"","modalities":["text"],"model":"LLaMA 33B OASST SFT","model_family":"LLaMA","model_variant":"33B OASST SFT","model_version":"LLaMA OASST","name":"LLaMA 33B OASST SFT","open_source":true,"parameter_size":"33B","provider":"openassistant","url":""},"llama-3_1-70b-tfree-hat-sft":{"country":"DE","id":"llama-3_1-70b-tfree-hat-sft","launched":"2024-07-23","modalities":["text"],"model":"Llama 3.1 70B TFree HAT","model_family":"Llama TFree HAT","model_variant":"3.1-70B TFree HAT SFT","model_version":"Llama TFree HAT","name":"Llama-3_1-70B-TFree-HAT-SFT","open_source":true,"parameter_size":"69.3B","provider":"alephalpha","url":""},"llama-4":{"country":"US","id":"llama-4","launched":"","modalities":["text"],"model":"Llama 4","model_family":"Llama","model_variant":"4","model_version":"Llama 4","name":"Llama 4","open_source":false,"parameter_size":"","provider":"meta","url":""},"llama-4-behemoth":{"country":"US","id":"llama-4-behemoth","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Behemoth","model_family":"Llama","model_variant":"Llama 4 Behemoth","model_version":"Llama 4","name":"Llama 4 Behemoth","open_source":false,"parameter_size":"","provider":"meta","url":""},"llama-4-maverick":{"country":"US","id":"llama-4-maverick","launched":"2025-04-05","modalities":["text","visual"],"model":"Llama 4 Maverick 17B 128E","model_family":"Llama","model_variant":"Llama4","model_version":"Llama 4","name":"Llama 4 Maverick","open_source":true,"parameter_size":"400B","provider":"meta","url":""},"llama-4-maverick-17b-128e":{"active_parameters":"17B","country":"US","id":"llama-4-maverick-17b-128e","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Maverick 17B 128E","model_family":"Llama","model_variant":"Llama 4 Maverick 17B 128E Instruct","model_version":"Llama 4","name":"Llama 4 Maverick 17B 128E Instruct","open_source":true,"parameter_size":"400B","provider":"meta","url":""},"llama-4-maverick-17b-128e-base":{"active_parameters":"17B","country":"US","id":"llama-4-maverick-17b-128e-base","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Maverick 17B 128E","model_family":"Llama","model_variant":"Llama 4 Maverick 17B 128E Base","model_version":"Llama 4","name":"Llama 4 Maverick 17B 128E Base","open_source":true,"parameter_size":"400B","provider":"meta","url":""},"llama-4-maverick-17b-128e-instruct":{"active_parameters":"17B","country":"US","id":"llama-4-maverick-17b-128e-instruct","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Maverick","model_family":"Llama","model_variant":"Llama 4 Maverick 17B","model_version":"Llama 4","name":"Llama 4 Maverick 17B 128E Instruct","open_source":true,"parameter_size":"17B","provider":"meta","url":""},"llama-4-maverick-base":{"country":"US","id":"llama-4-maverick-base","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Maverick 17B 128E","model_family":"Llama","model_variant":"Llama4 Maverick Base","model_version":"Llama 4","name":"Llama 4 Maverick Base","open_source":true,"parameter_size":"400B","provider":"meta","url":""},"llama-4-memonet":{"country":"US","id":"llama-4-memonet","launched":"","modalities":["text"],"model":"Llama 4 MemoNet","model_family":"Llama","model_variant":"4 MemoNet","model_version":"Llama 4","name":"Llama 4 MemoNet","open_source":false,"parameter_size":"","provider":"meta","url":""},"llama-4-scout":{"country":"US","id":"llama-4-scout","launched":"","modalities":["text"],"model":"Llama 4 Scout","model_family":"Llama","model_variant":"Llama 4 Scout","model_version":"Llama 1","name":"Llama 4 Scout","open_source":true,"parameter_size":"","provider":"meta","url":""},"llama-4-scout-17b":{"country":"US","id":"llama-4-scout-17b","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Scout 17B 16E","model_family":"Llama","model_variant":"Llama4-Scout-17B","model_version":"Llama 4","name":"Llama 4 Scout 17B","open_source":true,"parameter_size":"17B","provider":"meta","url":""},"llama-4-scout-17b-16e":{"active_parameters":"17B","country":"US","id":"llama-4-scout-17b-16e","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Scout 17B 16E","model_family":"Llama","model_variant":"Llama4-Scout-17B-16E","model_version":"Llama 4","name":"Llama 4 Scout 17B 16E","open_source":true,"parameter_size":"109B","provider":"meta","url":""},"llama-4-scout-17b-16e-base":{"active_parameters":"17B","country":"US","id":"llama-4-scout-17b-16e-base","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Scout 17B 16E","model_family":"Llama","model_variant":"Llama4-Scout-17B-16E Base","model_version":"Llama 4","name":"Llama 4 Scout 17B 16E Base","open_source":true,"parameter_size":"109B","provider":"meta","url":""},"llama-4-scout-17b-16e-instruct":{"active_parameters":"17B","country":"US","id":"llama-4-scout-17b-16e-instruct","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Scout","model_family":"Llama","model_variant":"Llama 4 Scout 17B","model_version":"Llama 4","name":"Llama 4 Scout 17B 16E Instruct","open_source":true,"parameter_size":"17B","provider":"meta","url":""},"llama-4-scout-17b-base":{"country":"US","id":"llama-4-scout-17b-base","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Scout 17B 16E","model_family":"Llama","model_variant":"Llama4-Scout-17B","model_version":"Llama 4","name":"Llama 4 Scout 17B Base","open_source":true,"parameter_size":"17B","provider":"meta","url":""},"llama-c-34b":{"country":"US","id":"llama-c-34b","launched":"","modalities":["text"],"model":"Llama C 34B","model_family":"Llama","model_variant":"Llama C 34B","model_version":"Llama 1","name":"Llama C 34B","open_source":true,"parameter_size":"34B","provider":"meta","url":""},"llama-c-70b":{"country":"US","id":"llama-c-70b","launched":"","modalities":["text"],"model":"Llama C 70B","model_family":"Llama","model_variant":"Llama C 70B","model_version":"Llama 1","name":"Llama C 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama-nemotron-super-49b-v1.5":{"country":"US","id":"llama-nemotron-super-49b-v1.5","launched":"2025-10-10","modalities":["text"],"model":"Llama Nemotron Super","model_family":"Llama","model_variant":"Nemotron Super 49B v1.5","model_version":"Llama Nemotron v1.5","name":"Llama Nemotron Super 49B v1.5","open_source":true,"parameter_size":"49B","provider":"nvidia","url":""},"llama-nemotron-super-v1":{"country":"US","id":"llama-nemotron-super-v1","launched":"","modalities":["text"],"model":"Llama Nemotron Super","model_family":"Llama","model_variant":"Nemotron Super v1","model_version":"Llama Nemotron v1","name":"Llama-Nemotron-Super v1","open_source":true,"parameter_size":"49B","provider":"nvidia","url":""},"llama-spark-8b":{"country":"US","id":"llama-spark-8b","launched":"2024-07-26","modalities":["text"],"model":"Llama Spark","model_family":"Llama Spark","model_variant":"8B","model_version":"Llama Spark 8B","name":"Llama Spark 8B","open_source":true,"parameter_size":"8B","provider":"arcee","url":""},"llama-tfree-hat-pretrained-7b-dpo":{"country":"DE","id":"llama-tfree-hat-pretrained-7b-dpo","launched":"2025-07-31","modalities":["text"],"model":"Llama TFree HAT 7B","model_family":"Llama TFree HAT","model_variant":"TFree HAT Pretrained 7B DPO","model_version":"Llama TFree HAT","name":"Llama-TFree-HAT-Pretrained-7B-DPO","open_source":true,"parameter_size":"7.19B","provider":"alephalpha","url":""},"llama2-70b-steerlm-chat":{"country":"US","id":"llama2-70b-steerlm-chat","launched":"2023-06-18","modalities":["text"],"model":"Llama 2 70B","model_family":"Llama","model_variant":"Llama 2 70B SteerLM","model_version":"Llama 2","name":"Llama2 70B SteerLM Chat","open_source":true,"parameter_size":"70B","provider":"nvidia","url":""},"llama3-3.2b":{"country":"US","id":"llama3-3.2b","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 3.2B","model_family":"Llama","model_variant":"Llama3-3.2B","model_version":"Llama 3","name":"Llama3-3.2B","open_source":true,"parameter_size":"3.2B","provider":"meta","url":""},"llama3-70b-instruct":{"country":"US","id":"llama3-70b-instruct","launched":"","modalities":["text"],"model":"Llama3-70B-instruct","model_family":"Llama3-70B-instruct","model_variant":"Llama3-70B-instruct","model_version":"Llama3-70B-instruct","name":"Llama3-70B-instruct","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama3-8b":{"country":"US","id":"llama3-8b","launched":"2024-04-18","modalities":["text"],"model":"Llama 3 8B","model_family":"Llama","model_variant":"Llama3 8B","model_version":"Llama 3","name":"Llama3 8B","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama3-8b-base":{"country":"US","id":"llama3-8b-base","launched":"","modalities":["text"],"model":"Llama3-8B-base","model_family":"Llama3-8B-base","model_variant":"Llama3-8B-base","model_version":"Llama3-8B-base","name":"Llama3-8B-base","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama3-8b-instruct":{"country":"US","id":"llama3-8b-instruct","launched":"","modalities":["text"],"model":"Llama3-8B-instruct","model_family":"Llama3-8B-instruct","model_variant":"Llama3-8B-instruct","model_version":"Llama3-8B-instruct","name":"Llama3-8B-instruct","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"llama3.0-70b":{"country":"US","id":"llama3.0-70b","launched":"2024-04-18","modalities":["text"],"model":"Llama 3.0 70B","model_family":"Llama 3.0","model_variant":"70B","model_version":"Llama 3.0","name":"Llama 3.0 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama3.1-70b":{"country":"US","id":"llama3.1-70b","launched":"","modalities":["text"],"model":"Llama 3.1 70B","model_family":"Llama 3.1","model_variant":"70B","model_version":"Llama 3.1","name":"Llama 3.1 70B","open_source":true,"parameter_size":"70B","provider":"meta","url":""},"llama4-maverick-instruct-basic":{"country":"US","id":"llama4-maverick-instruct-basic","launched":"2025-04-05","modalities":["text"],"model":"Llama 4 Maverick","model_family":"Llama","model_variant":"Llama4 Maverick Instruct Basic","model_version":"Llama 4","name":"Llama4 Maverick Instruct Basic","open_source":true,"parameter_size":"","provider":"meta","url":""},"llava-1.5-13b":{"country":"US","id":"llava-1.5-13b","launched":"","modalities":["visual"],"model":"LLaVA-1.5-13B","model_family":"LLaVA","model_variant":"1.5-13B","model_version":"LLaVA 1.5","name":"LLaVA-1.5-13B","open_source":"plus","parameter_size":"13B","provider":"microsoft","url":""},"llava-1.5-7b":{"country":"US","id":"llava-1.5-7b","launched":"","modalities":["visual"],"model":"LLaVA-1.5-7B","model_family":"LLaVA","model_variant":"1.5-7B","model_version":"LLaVA 1.5","name":"LLaVA-1.5-7B","open_source":"plus","parameter_size":"7B","provider":"microsoft","url":""},"llava-1.6-13b":{"country":"US","id":"llava-1.6-13b","launched":"","modalities":["visual"],"model":"LLaVA-1.6-13B","model_family":"LLaVA","model_variant":"1.6-13B","model_version":"LLaVA 1.6","name":"LLaVA-1.6-13B","open_source":true,"parameter_size":"13B","provider":"microsoft","url":"https://llava-vl.github.io/"},"llava-1.6-34b":{"country":"US","id":"llava-1.6-34b","launched":"","modalities":["visual"],"model":"LLaVA-1.6-34B","model_family":"LLaVA","model_variant":"1.6-34B","model_version":"LLaVA 1.6","name":"LLaVA-1.6-34B","open_source":true,"parameter_size":"34B","provider":"microsoft","url":"https://llava-vl.github.io/"},"llava-1.6-7b":{"country":"US","id":"llava-1.6-7b","launched":"","modalities":["visual"],"model":"LLaVA-1.6-7B","model_family":"LLaVA","model_variant":"1.6-7B","model_version":"LLaVA 1.6","name":"LLaVA-1.6-7B","open_source":true,"parameter_size":"7B","provider":"microsoft","url":"https://llava-vl.github.io/"},"llava-next-34b":{"country":"CN","id":"llava-next-34b","launched":"","modalities":["visual"],"model":"LLaVA-NeXT 34B","model_family":"LLaVA","model_variant":"LLaVA-NeXT 34B","model_version":"LLaVA-NeXT","name":"LLaVA-NeXT 34B","open_source":true,"parameter_size":"34B","provider":"bytedance","url":""},"llava-onevision-72b":{"country":"US","id":"llava-onevision-72b","launched":"2024-08-06","modalities":["visual"],"model":"LLaVA OneVision 72B","model_family":"LLaVA","model_variant":"OneVision-72B","model_version":"LLaVA OneVision","name":"LLaVA OneVision-72B","open_source":"plus","parameter_size":"72B","provider":"microsoft","url":""},"llava-onevision-7b":{"country":"US","id":"llava-onevision-7b","launched":"","modalities":["visual"],"model":"LLaVA OneVision 7B","model_family":"LLaVA","model_variant":"OneVision-7B","model_version":"LLaVA OneVision","name":"LLaVA OneVision-7B","open_source":"plus","parameter_size":"7B","provider":"microsoft","url":""},"llava-onevision-qwen2-72b-ov":{"country":"US","id":"llava-onevision-qwen2-72b-ov","launched":"","modalities":["visual"],"model":"LLaVA OneVision Qwen2 72B","model_family":"LLaVA","model_variant":"OneVision Qwen2 72B OV","model_version":"LLaVA OneVision","name":"LLaVA OneVision Qwen2 72B OV","open_source":true,"parameter_size":"72B","provider":"llava","url":""},"longcat-flash":{"country":"CN","id":"longcat-flash","launched":"","modalities":["text"],"model":"LongCat-Flash","model_family":"LongCat","model_variant":"Longcat-flash","model_version":"LongCat-Flash","name":"LongCat-Flash","open_source":true,"parameter_size":"560B","provider":"meituan","url":""},"longcat-flash-chat":{"country":"CN","id":"longcat-flash-chat","launched":"","modalities":["text"],"model":"LongCat Flash","model_family":"LongCat","model_variant":"LongCat Flash Chat","model_version":"LongCat-Flash","name":"LongCat Flash Chat","open_source":true,"parameter_size":"","provider":"meituan","url":""},"longcat-flash-lite":{"active_parameters":"4.5B","country":"CN","id":"longcat-flash-lite","launched":"2026-01-28","modalities":["text"],"model":"LongCat-Flash-Lite","model_family":"LongCat","model_variant":"Flash-Lite","model_version":"LongCat-Flash-Lite","name":"LongCat-Flash-Lite","open_source":true,"parameter_size":"68.5B","provider":"meituan","url":"https://huggingface.co/meituan-longcat/LongCat-Flash-Lite"},"longcat-flash-thinking":{"country":"CN","id":"longcat-flash-thinking","launched":"","modalities":["text"],"model":"LongCat-Flash","model_family":"LongCat","model_variant":"Longcat-flash Thinking","model_version":"LongCat-Flash","name":"LongCat-Flash-Thinking","open_source":true,"parameter_size":"560B","provider":"meituan","url":""},"longcat-flash-thinking-2601":{"active_parameters":"27B","country":"CN","id":"longcat-flash-thinking-2601","launched":"2026-01-14","modalities":["text"],"model":"LongCat-Flash","model_family":"LongCat","model_variant":"Longcat-flash Thinking 2601","model_version":"LongCat-Flash","name":"LongCat-Flash-Thinking-2601","open_source":true,"parameter_size":"560B","provider":"meituan","url":""},"magicoder-s-cl-7b":{"country":"US","id":"magicoder-s-cl-7b","launched":"","modalities":["text"],"model":"Magicoder-S-CL-7B","model_family":"Magicoder-S-CL-7B","model_variant":"Magicoder-S-CL-7B","model_version":"Magicoder-S-CL-7B","name":"Magicoder-S-CL-7B","open_source":true,"parameter_size":"7B","provider":"magicoder","url":""},"magicoder-s-ds-6.7b":{"country":"US","id":"magicoder-s-ds-6.7b","launched":"","modalities":["text"],"model":"Magicoder-S-DS-6.7B","model_family":"Magicoder-S-DS-6.7B","model_variant":"Magicoder-S-DS-6.7B","model_version":"Magicoder-S-DS-6.7B","name":"Magicoder-S-DS-6.7B","open_source":true,"parameter_size":"6.7B","provider":"magicoder","url":""},"magistral-medium":{"country":"US","id":"magistral-medium","launched":"2025-06-10","modalities":["text"],"model":"Magistral medium","model_family":"Magistral","model_variant":"medium","model_version":"Magistral","name":"Magistral-medium","open_source":false,"parameter_size":"","provider":"magistral","url":""},"magistral-medium-1.0":{"country":"FR","id":"magistral-medium-1.0","launched":"2025-06-10","modalities":["text"],"model":"Magistral Medium 1.0","model_family":"Magistral","model_variant":"Medium 1.0","model_version":"Magistral 1.0","name":"Magistral Medium 1.0","open_source":false,"parameter_size":"","provider":"mistral","url":""},"magistral-medium-1.1":{"country":"FR","id":"magistral-medium-1.1","launched":"2025-07-24","modalities":["text"],"model":"Magistral Medium 1.1","model_family":"Magistral","model_variant":"Medium 1.1","model_version":"Magistral 1.1","name":"Magistral Medium 1.1","open_source":false,"parameter_size":"","provider":"mistral","url":""},"magistral-medium-1.2":{"country":"FR","id":"magistral-medium-1.2","launched":"2025-09-18","modalities":["text"],"model":"Magistral Medium 1.2","model_family":"Magistral","model_variant":"Medium 1.2","model_version":"Magistral 1.2","name":"Magistral Medium 1.2","open_source":false,"parameter_size":"","provider":"mistral","url":""},"magistral-medium-2506":{"country":"FR","id":"magistral-medium-2506","launched":"2025-06-08","modalities":["text"],"model":"Magistral Medium 2506","model_family":"Magistral","model_variant":"Medium 2506","model_version":"Magistral 2506","name":"Magistral Medium 2506","open_source":false,"parameter_size":"","provider":"mistral","url":""},"magistral-medium-thinking":{"country":"FR","id":"magistral-medium-thinking","launched":"","modalities":["text"],"model":"Magistral Medium","model_family":"Magistral","model_variant":"Medium Thinking","model_version":"Magistral","name":"Magistral Medium (Thinking)","open_source":false,"parameter_size":"","provider":"mistral","url":""},"magistral-small-1.0":{"country":"FR","id":"magistral-small-1.0","launched":"2025-06-10","modalities":["text"],"model":"Magistral Small 1.0","model_family":"Magistral","model_variant":"Small 1.0","model_version":"Magistral 1.0","name":"Magistral Small 1.0","open_source":true,"parameter_size":"","provider":"mistral","url":""},"magistral-small-1.1":{"country":"FR","id":"magistral-small-1.1","launched":"2025-07-24","modalities":["text"],"model":"Magistral Small 1.1","model_family":"Magistral","model_variant":"Small 1.1","model_version":"Magistral 1.1","name":"Magistral Small 1.1","open_source":true,"parameter_size":"","provider":"mistral","url":""},"magistral-small-1.2":{"country":"FR","id":"magistral-small-1.2","launched":"2025-09-17","modalities":["text"],"model":"Magistral Small 1.2","model_family":"Magistral","model_variant":"Small 1.2","model_version":"Magistral 1.2","name":"Magistral Small 1.2","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"magistral-small-2506":{"country":"FR","id":"magistral-small-2506","launched":"2025-06-08","modalities":["text"],"model":"Magistral Small 2506","model_family":"Magistral","model_variant":"Small 2506","model_version":"Magistral 2506","name":"Magistral Small 2506","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"magistral-small-2509":{"country":"FR","id":"magistral-small-2509","launched":"2025-09-17","modalities":["text"],"model":"Magistral Small 1.2","model_family":"Magistral","model_variant":"Small 2509","model_version":"Magistral 1.2","name":"Magistral Small 2509","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"magpie-8b":{"country":"US","id":"magpie-8b","launched":"2024-09-17","modalities":["text"],"model":"Magpie 8B","model_family":"Magpie","model_variant":"Magpie 8B","model_version":"Magpie","name":"Magpie 8B","open_source":"plus","parameter_size":"8B","provider":"allenai","url":""},"mai-1-preview":{"country":"FR","id":"mai-1-preview","launched":"","modalities":["text"],"model":"MAI-1","model_family":"MAI","model_variant":"MAI-1 Preview","model_version":"MAI-1 Preview","name":"MAI-1 Preview","open_source":false,"parameter_size":"","provider":"mistral","url":""},"manzano-30b":{"country":"US","id":"manzano-30b","launched":"","modalities":["visual"],"model":"Manzano 30B","model_family":"Manzano","model_variant":"Manzano 30B","model_version":"Manzano","name":"Manzano 30B","open_source":true,"parameter_size":"30B","provider":"apple","url":""},"manzano-3b":{"country":"US","id":"manzano-3b","launched":"2025-09-25","modalities":["visual"],"model":"Manzano 3B","model_family":"Manzano","model_variant":"Manzano 3B","model_version":"Manzano","name":"Manzano 3B","open_source":true,"parameter_size":"3B","provider":"apple","url":""},"marin-32b-bison":{"country":"US","id":"marin-32b-bison","launched":"2025-10-25","modalities":["text"],"model":"Marin 32B Bison","model_family":"Marin","model_variant":"32B Bison","model_version":"Marin 32B","name":"Marin-32B-Bison","open_source":true,"parameter_size":"32B","provider":"marin","url":""},"marin-32b-mantis":{"country":"US","id":"marin-32b-mantis","launched":"2025-10-25","modalities":["text"],"model":"Marin 32B Mantis","model_family":"Marin","model_variant":"32B Mantis","model_version":"Marin 32B","name":"Marin-32B-Mantis","open_source":true,"parameter_size":"32B","provider":"marin","url":""},"marker":{"country":"CN","id":"marker","launched":"","modalities":["visual"],"model":"Marker","model_family":"Marker","model_variant":"Base","model_version":"Marker","name":"Marker","open_source":true,"parameter_size":"","provider":"datalab","url":""},"marker-1.8.2":{"country":"US","id":"marker-1.8.2","launched":"","modalities":["visual"],"model":"Marker","model_family":"Marker","model_variant":"1.8.2","model_version":"Marker 1.8","name":"Marker 1.8.2","open_source":true,"parameter_size":"","provider":"datalab","url":""},"marker-v1.7.5":{"country":"CN","id":"marker-v1.7.5","launched":"","modalities":["visual"],"model":"Marker v1.7.5","model_family":"Marker","model_variant":"v1.7.5","model_version":"Marker","name":"Marker v1.7.5","open_source":true,"parameter_size":"","provider":"datalab","url":""},"mathpix":{"country":"US","id":"mathpix","launched":"","modalities":["visual"],"model":"Mathpix","model_family":"Mathpix","model_variant":"Base","model_version":"Mathpix","name":"Mathpix","open_source":true,"parameter_size":"","provider":"mathpix","url":""},"mathstral-7b":{"country":"FR","id":"mathstral-7b","launched":"2024-07-16","modalities":["text"],"model":"Mathstral 7B","model_family":"Mathstral","model_variant":"Mathstral 7B","model_version":"Mathstral","name":"Mathstral 7B","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"meta-llama-3-8b-instruct":{"country":"US","id":"meta-llama-3-8b-instruct","launched":"2024-04-18","modalities":["text"],"model":"Meta Llama 3 8B Instruct","model_family":"Llama","model_variant":"Llama 3 8B Instruct","model_version":"Llama 3","name":"Meta Llama 3 8B Instruct","open_source":true,"parameter_size":"8B","provider":"meta","url":""},"midm-2.0-base":{"country":"KR","id":"midm-2.0-base","launched":"2025-07-03","modalities":["text"],"model":"Mi:dm 2.0 Base","model_family":"Mi:dm","model_variant":"2.0 Base","model_version":"Mi:dm 2.0","name":"Mi:dm 2.0 Base","open_source":false,"parameter_size":"","provider":"kt","url":""},"midm-2.0-base-instruct":{"country":"KR","id":"midm-2.0-base-instruct","launched":"2025-07-03","modalities":["text"],"model":"Mi:dm 2.0 Base","model_family":"Mi:dm","model_variant":"2.0 Base Instruct","model_version":"Mi:dm 2.0","name":"Mi:dm 2.0 Base Instruct","open_source":false,"parameter_size":"","provider":"kt","url":""},"midm-2.0-mini":{"country":"KR","id":"midm-2.0-mini","launched":"2025-07-03","modalities":["text"],"model":"Mi:dm 2.0 Mini","model_family":"Mi:dm","model_variant":"2.0 Mini","model_version":"Mi:dm 2.0","name":"Mi:dm 2.0 Mini","open_source":false,"parameter_size":"","provider":"kt","url":""},"midm-2.0-mini-instruct":{"country":"KR","id":"midm-2.0-mini-instruct","launched":"2025-07-03","modalities":["text"],"model":"Mi:dm 2.0 Mini","model_family":"Mi:dm","model_variant":"2.0 Mini Instruct","model_version":"Mi:dm 2.0","name":"Mi:dm 2.0 Mini Instruct","open_source":false,"parameter_size":"","provider":"kt","url":""},"midm-k-2.5-pro":{"country":"KR","id":"midm-k-2.5-pro","launched":"2026-01-06","modalities":["text"],"model":"Mi:dm K 2.5 Pro","model_family":"Mi:dm","model_variant":"K 2.5 Pro","model_version":"Mi:dm K 2.5","name":"Mi:dm K 2.5 Pro","open_source":false,"parameter_size":"","provider":"kt","url":""},"mimo-7b-rl":{"country":"CN","id":"mimo-7b-rl","launched":"2025-04-30","modalities":["text"],"model":"MiMo 7B RL","model_family":"MiMo","model_variant":"7B-RL","model_version":"MiMo 7B","name":"MiMo-7B-RL","open_source":true,"parameter_size":"7B","provider":"xiaomi","url":""},"mimo-v2-flash":{"active_parameters":"15B","country":"CN","id":"mimo-v2-flash","launched":"2025-12-16","modalities":["text"],"model":"MiMo V2 Flash","model_family":"MiMo","model_variant":"V2-Flash","model_version":"MiMo V2","name":"MiMo V2 Flash","open_source":true,"parameter_size":"309B","provider":"xiaomi","url":""},"mimo-v2-flash-base":{"active_parameters":"15B","country":"CN","id":"mimo-v2-flash-base","launched":"2025-12-16","modalities":["text"],"model":"MiMo V2 Flash Base","model_family":"MiMo","model_variant":"V2-Flash Base","model_version":"MiMo V2","name":"MiMo V2 Flash Base","open_source":true,"parameter_size":"309B","provider":"xiaomi","url":""},"mimo-vl":{"country":"CN","id":"mimo-vl","launched":"2025-06-04","modalities":["visual"],"model":"MiMo VL 7B","model_family":"MiMo","model_variant":"MiMo-VL","model_version":"MiMo VL","name":"MiMo-VL","open_source":true,"parameter_size":"7B","provider":"xiaomi","url":""},"mimo-vl-7b-rl":{"country":"CN","id":"mimo-vl-7b-rl","launched":"2025-06-04","modalities":["visual"],"model":"MiMo VL 7B","model_family":"MiMo","model_variant":"MiMo-VL-7B","model_version":"MiMo VL","name":"MiMo-VL 7B-RL","open_source":true,"parameter_size":"7B","provider":"xiaomi","url":""},"mimo-vl-7b-rl-2508":{"country":"CN","id":"mimo-vl-7b-rl-2508","launched":"2025-08-08","modalities":["visual"],"model":"MiMo VL 7B","model_family":"MiMo","model_variant":"MiMo-VL-RL-2508","model_version":"MiMo VL","name":"MiMo-VL-RL-2508 (7B)","open_source":true,"parameter_size":"7B","provider":"xiaomi","url":""},"mimo-vl-7b-sft":{"country":"CN","id":"mimo-vl-7b-sft","launched":"2025-06-04","modalities":["visual"],"model":"MiMo VL 7B","model_family":"MiMo","model_variant":"MiMo-VL-7B","model_version":"MiMo VL","name":"MiMo-VL 7B-SFT","open_source":true,"parameter_size":"7B","provider":"xiaomi","url":""},"mineru-1.3.10":{"country":"CN","id":"mineru-1.3.10","launched":"","modalities":["visual"],"model":"MinerU v1.3.10","model_family":"MinerU","model_variant":"1.3.10","model_version":"MinerU","name":"MinerU v1.3.10","open_source":true,"parameter_size":"","provider":"opendatalab","url":""},"mineru-2.1.1":{"country":"CN","id":"mineru-2.1.1","launched":"","modalities":["visual"],"model":"MinerU 2.1.1","model_family":"MinerU","model_variant":"2.1.1","model_version":"MinerU","name":"MinerU 2.1.1","open_source":true,"parameter_size":"","provider":"opendatalab","url":""},"mineru-2.5.4":{"country":"CN","id":"mineru-2.5.4","launched":"","modalities":["visual"],"model":"MinerU 2.5.4","model_family":"MinerU","model_variant":"2.5.4","model_version":"MinerU","name":"MinerU 2.5.4","open_source":true,"parameter_size":"","provider":"opendatalab","url":""},"mineru2-pp":{"country":"CN","id":"mineru2-pp","launched":"","modalities":["visual"],"model":"MinerU2","model_family":"MinerU","model_variant":"MinerU2-pp","model_version":"MinerU2","name":"MinerU2-pp","open_source":true,"parameter_size":"","provider":"opendatalab","url":""},"mineru2-vlm":{"country":"CN","id":"mineru2-vlm","launched":"","modalities":["visual"],"model":"MinerU2-VLM","model_family":"MinerU","model_variant":"MinerU2-VLM","model_version":"MinerU2","name":"MinerU2-VLM","open_source":true,"parameter_size":"","provider":"opendatalab","url":""},"mineru2.0":{"country":"CN","id":"mineru2.0","launched":"","modalities":["visual"],"model":"MinerU2.0","model_family":"MinerU","model_variant":"2.0","model_version":"MinerU","name":"MinerU2.0","open_source":true,"parameter_size":"1.2B","provider":"opendatalab","url":""},"mineru2.5":{"country":"CN","id":"mineru2.5","launched":"","modalities":["visual"],"model":"MinerU2.5","model_family":"MinerU","model_variant":"MinerU2.5","model_version":"MinerU2.5","name":"MinerU2.5","open_source":true,"parameter_size":"","provider":"opendatalab","url":""},"mini-omni2":{"country":"CN","id":"mini-omni2","launched":"2024-10-15","modalities":["text","audio"],"model":"Mini-Omni2","model_family":"Mini-Omni","model_variant":"Mini-Omni2","model_version":"Mini-Omni2","name":"Mini-Omni2","open_source":true,"parameter_size":"0.6B","provider":"gpt4o-omni","url":""},"minicpm-sala":{"country":"CN","id":"minicpm-sala","launched":"2026-02-11","modalities":["text"],"model":"MiniCPM-SALA","model_family":"MiniCPM","model_variant":"MiniCPM-SALA","model_version":"MiniCPM SALA","name":"MiniCPM-SALA","open_source":true,"parameter_size":"9B","provider":"openbmb","url":""},"minicpm-v-2.0-3b":{"country":"CN","id":"minicpm-v-2.0-3b","launched":"2024-04-23","modalities":["visual","text"],"model":"MiniCPM-V 2.0 3B","model_family":"MiniCPM","model_variant":"MiniCPM-V 2.0 3B","model_version":"MiniCPM V 2.0","name":"MiniCPM-V 2.0 3B","open_source":true,"parameter_size":"3B","provider":"bytedance","url":""},"minicpm-v-2_6":{"country":"CN","id":"minicpm-v-2_6","launched":"","modalities":["visual"],"model":"MiniCPM-V 2.6","model_family":"MiniCPM","model_variant":"MiniCPM-V 2.6","model_version":"MiniCPM V 2.6","name":"MiniCPM-V 2.6","open_source":true,"parameter_size":"","provider":"openbmb","url":""},"minicpm-v-4.5-8b":{"country":"CN","id":"minicpm-v-4.5-8b","launched":"","modalities":["visual"],"model":"MiniCPM-V 4.5 8B","model_family":"MiniCPM","model_variant":"MiniCPM-V 4.5 8B","model_version":"MiniCPM V 4.5","name":"MiniCPM-V 4.5 8B","open_source":true,"parameter_size":"8B","provider":"openbmb","url":""},"minicpm4-0.5b":{"country":"CN","id":"minicpm4-0.5b","launched":"2025-06-06","modalities":["visual","text"],"model":"MiniCPM4 0.5B","model_family":"MiniCPM","model_variant":"MiniCPM4-0.5B","model_version":"MiniCPM 4","name":"MiniCPM4 0.5B","open_source":true,"parameter_size":"0.5B","provider":"bytedance","url":"https://huggingface.co/openbmb/MiniCPM4-0.5B"},"minicpm4.1-8b":{"country":"CN","id":"minicpm4.1-8b","launched":"2025-09-05","modalities":["text"],"model":"MiniCPM-4.1 8B","model_family":"MiniCPM","model_variant":"MiniCPM-4.1 8B","model_version":"MiniCPM 4.1","name":"MiniCPM-4.1 8B","open_source":true,"parameter_size":"8B","provider":"openbmb","url":""},"minimax-m1":{"country":"CN","id":"minimax-m1","launched":"2025-06-14","modalities":["text"],"model":"MiniMax M1","model_family":"MiniMax","model_variant":"M1","model_version":"MiniMax M1","name":"MiniMax M1","open_source":true,"parameter_size":"456B","provider":"minimax","url":""},"minimax-m1-40k":{"country":"CN","id":"minimax-m1-40k","launched":"2025-06-14","modalities":["text"],"model":"MiniMax M1 40K","model_family":"MiniMax","model_variant":"M1 40K","model_version":"MiniMax M1","name":"MiniMax M1 40K","open_source":true,"parameter_size":"456B","provider":"minimax","url":""},"minimax-m1-80k":{"country":"CN","id":"minimax-m1-80k","launched":"2025-06-14","modalities":["text"],"model":"MiniMax M1 80K","model_family":"MiniMax","model_variant":"M1 80K","model_version":"MiniMax M1","name":"MiniMax M1 80K","open_source":true,"parameter_size":"456B","provider":"minimax","url":""},"minimax-m2":{"country":"CN","id":"minimax-m2","launched":"2025-10-23","modalities":["text"],"model":"MiniMax M2","model_family":"MiniMax","model_variant":"M2","model_version":"MiniMax M2","name":"MiniMax M2","open_source":true,"parameter_size":"229B","provider":"minimax","url":""},"minimax-m2.1":{"country":"CN","id":"minimax-m2.1","launched":"2025-12-21","modalities":["text"],"model":"MiniMax M2.1","model_family":"MiniMax","model_variant":"M2.1","model_version":"MiniMax M2.1","name":"MiniMax M2.1","open_source":true,"parameter_size":"229B","provider":"minimax","url":""},"minimax-m2.5":{"country":"CN","id":"minimax-m2.5","launched":"2026-02-12","modalities":["text"],"model":"MiniMax M2.5","model_family":"MiniMax","model_variant":"M2.5","model_version":"MiniMax M2.5","name":"MiniMax M2.5","open_source":true,"parameter_size":"","provider":"minimax","url":""},"ministral-3-14b-base":{"country":"FR","id":"ministral-3-14b-base","launched":"2025-12-02","modalities":["text"],"model":"Ministral 3 14B","model_family":"Ministral","model_variant":"Ministral 3 14B Base","model_version":"Ministral 3","name":"Ministral 3 14B Base","open_source":true,"parameter_size":"14B","provider":"mistral","url":""},"ministral-3-3b-base":{"country":"FR","id":"ministral-3-3b-base","launched":"2025-12-02","modalities":["text"],"model":"Ministral 3 3B","model_family":"Ministral","model_variant":"Ministral 3 3B Base","model_version":"Ministral 3","name":"Ministral 3 3B Base","open_source":true,"parameter_size":"3B","provider":"mistral","url":""},"ministral-3-8b-base":{"country":"FR","id":"ministral-3-8b-base","launched":"2025-12-02","modalities":["text"],"model":"Ministral 3 8B","model_family":"Ministral","model_variant":"Ministral 3 8B Base","model_version":"Ministral 3","name":"Ministral 3 8B Base","open_source":true,"parameter_size":"8B","provider":"mistral","url":""},"ministral-3-r-8b":{"country":"FR","id":"ministral-3-r-8b","launched":"","modalities":["text"],"model":"Ministral 3 R 8B","model_family":"Ministral","model_variant":"Ministral 3 R 8B","model_version":"Ministral 3","name":"Ministral-3-R 8B","open_source":true,"parameter_size":"8B","provider":"mistral","url":""},"ministral-3b-2410":{"country":"FR","id":"ministral-3b-2410","launched":"2024-10-16","modalities":["text"],"model":"Ministral 8B","model_family":"Ministral","model_variant":"Ministral 3B 24.10","model_version":"Ministral","name":"Ministral 3B (24.10)","open_source":true,"parameter_size":"3B","provider":"mistral","url":""},"ministral-8b":{"country":"FR","id":"ministral-8b","launched":"2024-10-16","modalities":["text"],"model":"Ministral 8B","model_family":"Ministral","model_variant":"Ministral-8B","model_version":"Ministral","name":"Ministral 8B","open_source":true,"parameter_size":"8B","provider":"mistral","url":""},"ministral-8b-2410":{"country":"FR","id":"ministral-8b-2410","launched":"2024-10-01","modalities":["text"],"model":"Ministral 8B","model_family":"Ministral","model_variant":"Ministral 8B 24.10","model_version":"Ministral","name":"Ministral 8B (24.10)","open_source":true,"parameter_size":"8B","provider":"mistral","url":""},"ministral-8b-instruct":{"country":"FR","id":"ministral-8b-instruct","launched":"","modalities":["text"],"model":"Ministral 8B","model_family":"Ministral","model_variant":"Ministral 8B Instruct","model_version":"Ministral","name":"Ministral 8B Instruct","open_source":true,"parameter_size":"8B","provider":"mistral","url":""},"mirothinker-v0.2-32b":{"country":"US","id":"mirothinker-v0.2-32b","launched":"2025-09-07","modalities":["text"],"model":"MiroThinker v0.2 32B","model_family":"MiroThinker","model_variant":"v0.2-32B","model_version":"MiroThinker v0.2","name":"MiroThinker-v0.2-32B","open_source":true,"parameter_size":"32B","provider":"miromind","url":""},"mirothinker-v1.0-72b":{"country":"US","id":"mirothinker-v1.0-72b","launched":"2025-11-13","modalities":["text"],"model":"MiroThinker v1.0 72B","model_family":"MiroThinker","model_variant":"v1.0-72B","model_version":"MiroThinker v1.0","name":"MiroThinker-v1.0-72B","open_source":true,"parameter_size":"72B","provider":"miromind","url":""},"mirothinker-v1.5-235b":{"country":"US","id":"mirothinker-v1.5-235b","launched":"2026-01-04","modalities":["text"],"model":"MiroThinker v1.5 235B","model_family":"MiroThinker","model_variant":"v1.5-235B","model_version":"MiroThinker v1.5","name":"MiroThinker-v1.5-235B","open_source":true,"parameter_size":"235B","provider":"miromind","url":""},"mirothinker-v1.5-30b":{"country":"US","id":"mirothinker-v1.5-30b","launched":"2026-01-04","modalities":["text"],"model":"MiroThinker v1.5 30B","model_family":"MiroThinker","model_variant":"v1.5-30B","model_version":"MiroThinker v1.5","name":"MiroThinker-v1.5-30B","open_source":true,"parameter_size":"30B","provider":"miromind","url":""},"mistra-medium":{"country":"FR","id":"mistra-medium","launched":"","modalities":["text"],"model":"Mistral Medium","model_family":"Mistral","model_variant":"Medium","model_version":"Mistral","name":"Mistral Medium","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-3.2-24b":{"country":"FR","id":"mistral-3.2-24b","launched":"2025-06-20","modalities":["text"],"model":"Mistral 3.2 24B","model_family":"Mistral","model_variant":"Mistral 3.2 24B","model_version":"Mistral 3.2","name":"Mistral-3.2-24B","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"mistral-7b":{"country":"FR","id":"mistral-7b","launched":"2023-09-27","modalities":["text"],"model":"Mistral 7B","model_family":"Mistral","model_variant":"Mistral-7B","model_version":"Mistral","name":"Mistral-7B","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-instruct":{"country":"FR","id":"mistral-7b-instruct","launched":"","modalities":["text"],"model":"Mistral 7B","model_family":"Mistral","model_variant":"Mistral 7B Instruct","model_version":"Mistral","name":"Mistral 7B Instruct","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-instruct-v0-2":{"country":"FR","id":"mistral-7b-instruct-v0-2","launched":"2023-12-12","modalities":["text"],"model":"Mistral 7B Instruct V0 2","model_family":"Mistral","model_variant":"Mistral 7B Instruct V0 2","model_version":"Mistral 0.2","name":"Mistral 7B Instruct V0 2","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-instruct-v0.1":{"country":"FR","id":"mistral-7b-instruct-v0.1","launched":"","modalities":["text"],"model":"Mistral 7B v0.1","model_family":"Mistral","model_variant":"Mistral-7B-Instruct-v0.1","model_version":"Mistral v0.1","name":"Mistral-7B-Instruct-v0.1","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-instruct-v0.2":{"country":"FR","id":"mistral-7b-instruct-v0.2","launched":"2023-12-12","modalities":["text"],"model":"Mistral 7B v0.2","model_family":"Mistral","model_variant":"Mistral-7B-Instruct-v0.2","model_version":"Mistral v0.2","name":"Mistral-7B-Instruct-v0.2","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-instruct-v0.3":{"country":"FR","id":"mistral-7b-instruct-v0.3","launched":"","modalities":["text"],"model":"Mistral 7B Instruct V0.3","model_family":"Mistral","model_variant":"Mistral 7B Instruct V0.3","model_version":"Mistral 0.3","name":"Mistral 7B Instruct V0.3","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-v0.1":{"country":"FR","id":"mistral-7b-v0.1","launched":"","modalities":["text"],"model":"Mistral 7B","model_family":"Mistral","model_variant":"Mistral-7B-v0.1","model_version":"Mistral v0.1","name":"Mistral-7B-v0.1","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-v0.2":{"country":"FR","id":"mistral-7b-v0.2","launched":"","modalities":["text"],"model":"Mistral 7B","model_family":"Mistral","model_variant":"Mistral-7B-v0.2","model_version":"Mistral v0.2","name":"Mistral-7B-v0.2","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-7b-v0.3":{"country":"FR","id":"mistral-7b-v0.3","launched":"","modalities":["text"],"model":"Mistral 7B","model_family":"Mistral","model_variant":"7B v0.3","model_version":"Mistral 0.3","name":"Mistral 7B v0.3","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-codealpaca-7b":{"country":"FR","id":"mistral-codealpaca-7b","launched":"","modalities":["text"],"model":"Mistral Codealpaca 7B","model_family":"Mistral","model_variant":"Mistral-codealpaca-7B","model_version":"Mistral Codealpaca","name":"Mistral-codealpaca-7B","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-instruct-7b":{"country":"FR","id":"mistral-instruct-7b","launched":"","modalities":["text"],"model":"Mistral Instruct 7B","model_family":"Mistral","model_variant":"Mistral Instruct 7B","model_version":"Mistral","name":"Mistral Instruct 7B","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mistral-large":{"country":"FR","id":"mistral-large","launched":"","modalities":["text"],"model":"Mistral Large","model_family":"Mistral","model_variant":"Mistral Large","model_version":"Mistral","name":"Mistral Large","open_source":true,"parameter_size":"","provider":"mistral","url":""},"mistral-large-2.1-2411":{"country":"FR","id":"mistral-large-2.1-2411","launched":"2024-11-01","modalities":["text"],"model":"Mistral Large 2.1","model_family":"Mistral","model_variant":"Mistral Large 2.1 2411","model_version":"Mistral 2.1","name":"Mistral Large 2.1 (2411)","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-large-2402":{"country":"FR","id":"mistral-large-2402","launched":"","modalities":["text"],"model":"Mistral Large 2402","model_family":"Mistral","model_variant":"Mistral Large 2402","model_version":"Mistral","name":"Mistral Large 2402","open_source":true,"parameter_size":"","provider":"mistral","url":""},"mistral-large-2407":{"country":"FR","id":"mistral-large-2407","launched":"2024-07-01","modalities":["text"],"model":"Mistral Large","model_family":"Mistral","model_variant":"Mistral Large","model_version":"Mistral","name":"Mistral Large (24-07)","open_source":false,"parameter_size":"123B","provider":"mistral","url":""},"mistral-large-2411":{"country":"FR","id":"mistral-large-2411","launched":"2024-11-01","modalities":["text"],"model":"Mistral Large","model_family":"Mistral","model_variant":"Mistral Large 24.11","model_version":"Mistral Large","name":"Mistral Large (24.11)","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-large-3":{"active_parameters":"41B","country":"FR","id":"mistral-large-3","launched":"2025-12-02","modalities":["text"],"model":"Mistral Large 3","model_family":"Mistral","model_variant":"Large 3","model_version":"Mistral 3","name":"Mistral Large 3","open_source":true,"parameter_size":"675B","provider":"mistral","url":""},"mistral-large-3-base":{"active_parameters":"","country":"FR","id":"mistral-large-3-base","launched":"2025-12-02","modalities":["text"],"model":"Mistral Large 3 Base","model_family":"Mistral","model_variant":"Large 3 Base","model_version":"Mistral 3","name":"Mistral Large 3 Base","open_source":true,"parameter_size":"","provider":"mistral","url":""},"mistral-large-instruct-2407":{"country":"FR","id":"mistral-large-instruct-2407","launched":"2024-07-01","modalities":["text"],"model":"Mistral Large Instruct 2407","model_family":"Mistral","model_variant":"Mistral Large Instruct 2407","model_version":"Mistral","name":"Mistral Large Instruct 2407","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-medium":{"country":"FR","id":"mistral-medium","launched":"2023-12-11","modalities":["text"],"model":"Mistral Medium","model_family":"Mistral","model_variant":"Mistral Medium","model_version":"Mistral","name":"Mistral Medium","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-medium-2505":{"country":"FR","id":"mistral-medium-2505","launched":"2025-05-01","modalities":["text"],"model":"Mistral Medium","model_family":"Mistral","model_variant":"Mistral Medium","model_version":"Mistral","name":"Mistral Medium (25-05)","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-medium-2508":{"country":"FR","id":"mistral-medium-2508","launched":"2025-08-01","modalities":["text"],"model":"Mistral Medium","model_family":"Mistral","model_variant":"Mistral Medium","model_version":"Mistral","name":"Mistral Medium (25-08)","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-medium-3":{"country":"FR","id":"mistral-medium-3","launched":"2025-05-07","modalities":["text"],"model":"Mistral Medium 3","model_family":"Mistral","model_variant":"Mistral Medium 3","model_version":"Mistral 3","name":"Mistral Medium 3","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-medium-3.1":{"country":"FR","id":"mistral-medium-3.1","launched":"2025-03-17","modalities":["text"],"model":"Mistral Medium 3.1","model_family":"Mistral","model_variant":"Mistral Medium 3.1","model_version":"Mistral 3.1","name":"Mistral Medium 3.1","open_source":false,"parameter_size":"50Be","provider":"mistral","url":""},"mistral-nemo-12b-instruct":{"country":"FR","id":"mistral-nemo-12b-instruct","launched":"","modalities":["text"],"model":"Mistral Nemo 12B","model_family":"Mistral","model_variant":"Mistral Nemo 12B Instruct","model_version":"Mistral","name":"Mistral Nemo 12B Instruct","open_source":true,"parameter_size":"12B","provider":"mistral","url":""},"mistral-nemo-instruct-2407":{"country":"FR","id":"mistral-nemo-instruct-2407","launched":"2024-07-01","modalities":["text"],"model":"Mistral Nemo Instruct 2407","model_family":"Mistral","model_variant":"Mistral Nemo Instruct 2407","model_version":"Mistral","name":"Mistral Nemo Instruct 2407","open_source":true,"parameter_size":"","provider":"mistral","url":""},"mistral-nemo-it-2407":{"country":"FR","id":"mistral-nemo-it-2407","launched":"","modalities":["text"],"model":"Mistral Nemo","model_family":"Mistral","model_variant":"Mistral Nemo IT 2407","model_version":"Mistral","name":"Mistral Nemo IT 2407","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-ocr-3":{"country":"FR","id":"mistral-ocr-3","launched":"2025-12-17","modalities":["visual"],"model":"Mistral OCR 3","model_family":"Mistral OCR","model_variant":"3","model_version":"Mistral OCR","name":"Mistral OCR 3","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-ocr-api":{"country":"FR","id":"mistral-ocr-api","launched":"","modalities":["visual"],"model":"Mistral OCR API","model_family":"Mistral OCR","model_variant":"API","model_version":"Mistral OCR","name":"Mistral OCR API","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-small":{"country":"FR","id":"mistral-small","launched":"","modalities":["text"],"model":"Mistral Small","model_family":"Mistral","model_variant":"Mistral Small","model_version":"Mistral","name":"Mistral Small","open_source":true,"parameter_size":"","provider":"mistral","url":""},"mistral-small-1.0":{"country":"FR","id":"mistral-small-1.0","launched":"","modalities":["text"],"model":"Mistral Small 1.0","model_family":"Mistral Small","model_variant":"1.0","model_version":"Mistral Small","name":"Mistral Small 1.0","open_source":false,"parameter_size":"","provider":"mistral","url":""},"mistral-small-2402":{"country":"FR","id":"mistral-small-2402","launched":"","modalities":["text"],"model":"Mistral Small 2402","model_family":"Mistral","model_variant":"Mistral Small 2402","model_version":"Mistral","name":"Mistral Small 2402","open_source":true,"parameter_size":"","provider":"mistral","url":""},"mistral-small-24b-instruct-2501":{"country":"FR","id":"mistral-small-24b-instruct-2501","launched":"","modalities":["text"],"model":"Mistral Small 24B Instruct 2501","model_family":"Mistral","model_variant":"Mistral Small 24B Instruct 2501","model_version":"Mistral","name":"Mistral Small 24B Instruct 2501","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"mistral-small-2506":{"country":"FR","id":"mistral-small-2506","launched":"2025-06-01","modalities":["text"],"model":"Mistral Small","model_family":"Mistral","model_variant":"Mistral Small","model_version":"Mistral","name":"Mistral Small (25-06)","open_source":true,"parameter_size":"22B","provider":"mistral","url":""},"mistral-small-3-24b-base":{"country":"FR","id":"mistral-small-3-24b-base","launched":"2025-06-20","modalities":["text"],"model":"Mistral Small 3 24B Base","model_family":"Mistral","model_variant":"Mistral-24B","model_version":"Mistral 3","name":"Mistral Small 3 24B Base","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"mistral-small-3.1-24b-instruct":{"country":"FR","id":"mistral-small-3.1-24b-instruct","launched":"2025-03-17","modalities":["text","visual"],"model":"Mistral Small 3.1 24B","model_family":"Mistral","model_variant":"Mistral-24B","model_version":"Mistral 3.1","name":"Mistral Small 3.1 24B Instruct","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"mistral-small-3.1-24b-instruct-2503":{"country":"FR","id":"mistral-small-3.1-24b-instruct-2503","launched":"2025-03-01","modalities":["text"],"model":"Mistral Small","model_family":"Mistral","model_variant":"Mistral Small 3.1","model_version":"Mistral 3.1","name":"Mistral Small 3.1 24B Instruct (25-03)","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"mistral-small-3.2-24b-instruct":{"country":"FR","id":"mistral-small-3.2-24b-instruct","launched":"2025-06-20","modalities":["text","visual"],"model":"Mistral Small 3.2 24B","model_family":"Mistral","model_variant":"Mistral-24B","model_version":"Mistral 3.2","name":"Mistral Small 3.2 24B Instruct","open_source":true,"parameter_size":"24B","provider":"mistral","url":""},"mistral-small-instruct-2409":{"country":"FR","id":"mistral-small-instruct-2409","launched":"","modalities":["text"],"model":"Mistral Small Instruct 2409","model_family":"Mistral","model_variant":"Mistral Small Instruct 2409","model_version":"Mistral","name":"Mistral Small Instruct 2409","open_source":true,"parameter_size":"","provider":"mistral","url":""},"mistralhermes-codepro-7b-v1":{"country":"FR","id":"mistralhermes-codepro-7b-v1","launched":"","modalities":["text"],"model":"MistralHermes CodePro 7B v1","model_family":"MistralHermes","model_variant":"MistralHermes-CodePro-7B-v1","model_version":"MistralHermes CodePro v1","name":"MistralHermes-CodePro-7B-v1","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mixtral-8x22b":{"country":"FR","id":"mixtral-8x22b","launched":"2024-04-17","modalities":["text"],"model":"Mixtral 8x22B","model_family":"Mixtral","model_variant":"Mixtral-8x22B","model_version":"Mixtral","name":"Mixtral 8x22B","open_source":true,"parameter_size":"141B","provider":"mistral","published_date":"2024-04-17","url":""},"mixtral-8x22b-instruct":{"country":"FR","id":"mixtral-8x22b-instruct","launched":"","modalities":["text"],"model":"Mixtral 8x 22B","model_family":"Mixtral","model_variant":"Mixtral 8x 22B Instruct","model_version":"Mixtral","name":"Mixtral 8x 22B Instruct","open_source":true,"parameter_size":"22B","provider":"mistral","url":""},"mixtral-8x22b-instruct-v0.1":{"country":"FR","id":"mixtral-8x22b-instruct-v0.1","launched":"","modalities":["text"],"model":"Mixtral 8x22B v0.1","model_family":"Mixtral","model_variant":"Mixtral-8x22B-Instruct-v0.1","model_version":"Mixtral v0.1","name":"Mixtral-8x22B-Instruct-v0.1","open_source":true,"parameter_size":"22B","provider":"mistral","url":""},"mixtral-8x7b":{"country":"FR","id":"mixtral-8x7b","launched":"2023-12-11","modalities":["text"],"model":"Mixtral 8x 7B","model_family":"Mixtral","model_variant":"Mixtral 8x 7B","model_version":"Mixtral","name":"Mixtral 8x 7B","open_source":true,"parameter_size":"7B","provider":"mistral","published_date":"2023-12-11","url":""},"mixtral-8x7b-instruct-v0.1":{"country":"FR","id":"mixtral-8x7b-instruct-v0.1","launched":"2023-12-11","modalities":["text"],"model":"Mixtral 8x7B v0.1","model_family":"Mixtral","model_variant":"Mixtral-8x7B-Instruct-v0.1","model_version":"Mixtral v0.1","name":"Mixtral-8x7B-Instruct-v0.1","open_source":true,"parameter_size":"7B","provider":"mistral","url":""},"mm1-30b":{"country":"US","id":"mm1-30b","launched":"","modalities":["visual"],"model":"MM1 30B","model_family":"MM1","model_variant":"MM1 30B","model_version":"MM1","name":"MM1 30B","open_source":true,"parameter_size":"30B","provider":"apple","url":""},"mm1.5-30b":{"country":"US","id":"mm1.5-30b","launched":"","modalities":["visual"],"model":"MM1.5 30B","model_family":"MM1","model_variant":"MM1.5 30B","model_version":"MM1.5","name":"MM1.5 30B","open_source":true,"parameter_size":"30B","provider":"apple","url":""},"mm1.5-3b":{"country":"US","id":"mm1.5-3b","launched":"2024-03-14","modalities":["visual"],"model":"MM1.5 3B","model_family":"MM1","model_variant":"MM1.5 3B","model_version":"MM1.5","name":"MM1.5 3B","open_source":true,"parameter_size":"3B","provider":"apple","url":""},"mobilellm-p1":{"country":"US","id":"mobilellm-p1","launched":"2025-10-17","modalities":["text"],"model":"MobileLLM P1","model_family":"MobileLLM","model_variant":"P1","model_version":"MobileLLM","name":"MobileLLM P1","open_source":true,"parameter_size":"1B","provider":"meta","url":""},"mobilellm-p1-base":{"country":"US","id":"mobilellm-p1-base","launched":"2025-10-17","modalities":["text"],"model":"MobileLLM P1","model_family":"MobileLLM","model_variant":"P1 Base","model_version":"MobileLLM P1","name":"MobileLLM P1 Base","open_source":"plus","parameter_size":"1B","provider":"meta","url":""},"mobilellm-r1-140m":{"country":"US","id":"mobilellm-r1-140m","launched":"","modalities":["text"],"model":"MobileLLM R1 140M","model_family":"MobileLLM","model_variant":"MobileLLM-R1-140M","model_version":"MobileLLM R1","name":"MobileLLM R1 140M","open_source":true,"parameter_size":"0.14B","provider":"meta","url":"https://huggingface.co/facebook/MobileLLM-R1-950M"},"mobilellm-r1-140m-base":{"country":"US","id":"mobilellm-r1-140m-base","launched":"2025-09-12","modalities":["text"],"model":"MobileLLM R1 950M","model_family":"MobileLLM","model_variant":"MobileLLM-R1-140M-Base","model_version":"MobileLLM R1","name":"MobileLLM R1 140M Base","open_source":"plus","parameter_size":"0.14B","provider":"meta","url":"https://huggingface.co/facebook/MobileLLM-R1-950M"},"mobilellm-r1-360m":{"country":"US","id":"mobilellm-r1-360m","launched":"","modalities":["text"],"model":"MobileLLM R1 360M","model_family":"MobileLLM","model_variant":"MobileLLM-R1-360M","model_version":"MobileLLM R1","name":"MobileLLM R1 360M","open_source":true,"parameter_size":"0.36B","provider":"meta","url":"https://huggingface.co/facebook/MobileLLM-R1-950M"},"mobilellm-r1-360m-base":{"country":"US","id":"mobilellm-r1-360m-base","launched":"","modalities":["text"],"model":"MobileLLM R1 950M","model_family":"MobileLLM","model_variant":"MobileLLM-R1-360M-Base","model_version":"MobileLLM R1","name":"MobileLLM R1 360M Base","open_source":"plus","parameter_size":"0.36B","provider":"meta","url":"https://huggingface.co/facebook/MobileLLM-R1-950M"},"mobilellm-r1-950m":{"country":"US","id":"mobilellm-r1-950m","launched":"2025-09-12","modalities":["text"],"model":"MobileLLM R1 950M","model_family":"MobileLLM","model_variant":"MobileLLM-R1-950M","model_version":"MobileLLM R1","name":"MobileLLM R1 950M","open_source":"plus","parameter_size":"0.95B","provider":"meta","url":"https://huggingface.co/facebook/MobileLLM-R1-950M"},"mobilellm-r1-950m-base":{"country":"US","id":"mobilellm-r1-950m-base","launched":"","modalities":["text"],"model":"MobileLLM R1 950M","model_family":"MobileLLM","model_variant":"MobileLLM-R1-950M-Base","model_version":"MobileLLM R1","name":"MobileLLM R1 950M Base","open_source":"plus","parameter_size":"0.95B","provider":"meta","url":"https://huggingface.co/facebook/MobileLLM-R1-950M"},"molmo-72b":{"country":"US","id":"molmo-72b","launched":"2024-09-25","modalities":["visual"],"model":"Molmo 72B","model_family":"Molmo","model_variant":"Molmo-72B","model_version":"Molmo","name":"Molmo-72B","open_source":"plus","parameter_size":"72B","provider":"allenai","url":""},"molmo-72b-0924":{"country":"US","id":"molmo-72b-0924","launched":"2024-09-01","modalities":["visual"],"model":"Molmo 72B","model_family":"Molmo","model_variant":"Molmo-72B 0924","model_version":"Molmo","name":"Molmo-72B (2024-09)","open_source":"plus","parameter_size":"72B","provider":"allenai","url":""},"molmo-7b-d":{"country":"US","id":"molmo-7b-d","launched":"2024-09-25","modalities":["visual"],"model":"Molmo 7B","model_family":"Molmo","model_variant":"Molmo-7B-D","model_version":"Molmo","name":"Molmo-7B-D","open_source":"plus","parameter_size":"7B","provider":"allenai","url":""},"molmo-7b-d-0924":{"country":"US","id":"molmo-7b-d-0924","launched":"2024-09-01","modalities":["visual"],"model":"Molmo 7B","model_family":"Molmo","model_variant":"Molmo-7B-D 0924","model_version":"Molmo","name":"Molmo-7B-D (2024-09)","open_source":"plus","parameter_size":"7B","provider":"allenai","url":""},"molmo-7b-o":{"country":"US","id":"molmo-7b-o","launched":"2024-09-25","modalities":["visual"],"model":"Molmo 7B","model_family":"Molmo","model_variant":"Molmo-7B-O","model_version":"Molmo","name":"Molmo-7B-O","open_source":"plus","parameter_size":"7B","provider":"allenai","url":""},"molmo2-4b":{"country":"US","id":"molmo2-4b","launched":"","modalities":["visual"],"model":"Molmo2 4B","model_family":"Molmo2","model_variant":"Molmo2-4B","model_version":"Molmo2","name":"Molmo2-4B","open_source":"plus","parameter_size":"4B","provider":"allenai","url":""},"molmo2-8b":{"country":"US","id":"molmo2-8b","launched":"","modalities":["visual"],"model":"Molmo2 8B","model_family":"Molmo2","model_variant":"Molmo2-8B","model_version":"Molmo2","name":"Molmo2-8B","open_source":"plus","parameter_size":"8B","provider":"allenai","url":""},"molmo2-o-7b":{"country":"US","id":"molmo2-o-7b","launched":"","modalities":["visual"],"model":"Molmo2 O 7B","model_family":"Molmo2","model_variant":"Molmo2-O-7B","model_version":"Molmo2","name":"Molmo2-O-7B","open_source":"plus","parameter_size":"7B","provider":"allenai","url":""},"molmoe-1b":{"country":"US","id":"molmoe-1b","launched":"","modalities":["visual"],"model":"MolmoE 1B","model_family":"Molmo","model_variant":"MolmoE-1B","model_version":"MolmoE","name":"MolmoE-1B","open_source":"plus","parameter_size":"1B","provider":"allenai","url":""},"monad":{"country":"FR","id":"monad","launched":"2025-11-10","modalities":["text"],"model":"Monad 56M","model_family":"Monad","model_variant":"56M","model_version":"Monad","name":"Monad 56M","open_source":"plus","parameter_size":"0.056B","provider":"pleias","url":""},"monkeyocr-1.2b":{"country":"CN","id":"monkeyocr-1.2b","launched":"","modalities":["visual"],"model":"MonkeyOCR 1.2B","model_family":"MonkeyOCR","model_variant":"1.2B","model_version":"MonkeyOCR","name":"MonkeyOCR 1.2B","open_source":true,"parameter_size":"1.2B","provider":"kingsoft","url":""},"monkeyocr-3b":{"country":"CN","id":"monkeyocr-3b","launched":"","modalities":["visual"],"model":"MonkeyOCR","model_family":"MonkeyOCR","model_variant":"3B","model_version":"MonkeyOCR","name":"MonkeyOCR-3B","open_source":true,"parameter_size":"3B","provider":"monkeyocr","url":""},"monkeyocr-pro-1.2b":{"country":"CN","id":"monkeyocr-pro-1.2b","launched":"","modalities":["visual"],"model":"MonkeyOCR Pro","model_family":"MonkeyOCR","model_variant":"Pro 1.2B","model_version":"MonkeyOCR","name":"MonkeyOCR-pro-1.2B","open_source":true,"parameter_size":"1.2B","provider":"monkeyocr","url":""},"monkeyocr-pro-3b":{"country":"CN","id":"monkeyocr-pro-3b","launched":"","modalities":["visual"],"model":"MonkeyOCR Pro 3B","model_family":"MonkeyOCR","model_variant":"Pro 3B","model_version":"MonkeyOCR Pro","name":"MonkeyOCR Pro 3B","open_source":true,"parameter_size":"3B","provider":"kingsoft","url":""},"moondream-9b-a2b":{"active_parameters":"2B","country":"US","id":"moondream-9b-a2b","launched":"2025-09-28","modalities":["visual"],"model":"Moondream 3","model_family":"Moondream","model_variant":"Moondream 3 Preview","model_version":"Moondream 3","name":"Moondream-9B-A2B","open_source":true,"parameter_size":"9B","provider":"moondream","url":""},"moonlight-base":{"active_parameters":"3B","country":"CN","id":"moonlight-base","launched":"2025-02-24","modalities":["text"],"model":"Moonlight Base","model_family":"Moonlight","model_variant":"Moonlight Base","model_version":"Moonlight","name":"Moonlight-Base","open_source":true,"parameter_size":"16B","provider":"moonshotai","url":""},"moonlight-instruct":{"active_parameters":"3B","country":"CN","id":"moonlight-instruct","launched":"2025-02-24","modalities":["text"],"model":"Moonlight Instruct","model_family":"Moonlight","model_variant":"Moonlight Instruct","model_version":"Moonlight","name":"Moonlight-Instruct","open_source":true,"parameter_size":"16B","provider":"moonshotai","url":""},"moshi":{"country":"FR","id":"moshi","launched":"2024-09-17","modalities":["text","audio"],"model":"Moshi","model_family":"Moshi","model_variant":"Moshi","model_version":"Moshi","name":"Moshi","open_source":true,"parameter_size":"7B","provider":"kyutai","url":""},"motif-2-12.7b-base":{"country":"KR","id":"motif-2-12.7b-base","launched":"2025-11-12","modalities":["text"],"model":"Motif-2 12.7B Base","model_family":"Motif","model_variant":"2-12.7B-Base","model_version":"Motif 2","name":"Motif-2-12.7B-Base","open_source":true,"parameter_size":"12.7B","provider":"motif","url":""},"motif-2-12.7b-instruct":{"country":"KR","id":"motif-2-12.7b-instruct","launched":"2025-11-12","modalities":["text"],"model":"Motif-2 12.7B Instruct","model_family":"Motif","model_variant":"2-12.7B-Instruct","model_version":"Motif 2","name":"Motif-2-12.7B-Instruct","open_source":true,"parameter_size":"12.7B","provider":"motif","url":""},"motif-2-12.7b-reasoning":{"country":"KR","id":"motif-2-12.7b-reasoning","launched":"2025-11-12","modalities":["text"],"model":"Motif-2 12.7B Reasoning","model_family":"Motif","model_variant":"2-12.7B-Reasoning","model_version":"Motif 2","name":"Motif-2-12.7B-Reasoning","open_source":true,"parameter_size":"12.7B","provider":"motif","url":""},"mpt-30b-chat":{"country":"US","id":"mpt-30b-chat","launched":"","modalities":["text"],"model":"MPT 30B","model_family":"MPT","model_variant":"MPT 30B Chat","model_version":"MPT","name":"MPT 30B Chat","open_source":true,"parameter_size":"30B","provider":"mosaicml","url":""},"mpt-7b-chat":{"country":"US","id":"mpt-7b-chat","launched":"","modalities":["text"],"model":"MPT 7B","model_family":"MPT","model_variant":"MPT 7B Chat","model_version":"MPT","name":"MPT 7B Chat","open_source":true,"parameter_size":"7B","provider":"mosaicml","url":""},"nanbeige4-3b-thinking-2511":{"country":"CN","id":"nanbeige4-3b-thinking-2511","launched":"2025-11-21","modalities":["text"],"model":"Nanbeige4 3B","model_family":"Nanbeige","model_variant":"4-3B Thinking 2511","model_version":"Nanbeige4","name":"Nanbeige4-3B-Thinking-2511","open_source":true,"parameter_size":"3B","provider":"boss","url":""},"nanbeige4.1-3b":{"country":"CN","id":"nanbeige4.1-3b","launched":"2026-02-11","modalities":["text"],"model":"Nanbeige4.1 3B","model_family":"Nanbeige","model_variant":"4.1-3B","model_version":"Nanbeige4.1","name":"Nanbeige4.1-3B","open_source":true,"parameter_size":"3B","provider":"boss","url":""},"nano-banana-2.0":{"country":"","id":"nano-banana-2.0","launched":"2025-11-25","modalities":["visual"],"model":"Nano Banana 2.0","model_family":"Nano Banana","model_variant":"2.0","model_version":"Nano Banana","name":"Nano Banana 2.0","open_source":false,"parameter_size":"","provider":"unknown","url":""},"nanonets-ocr-s":{"country":"US","id":"nanonets-ocr-s","launched":"","modalities":["visual"],"model":"Nanonets OCR S","model_family":"Nanonets OCR","model_variant":"S","model_version":"Nanonets OCR","name":"Nanonets OCR S","open_source":false,"parameter_size":"","provider":"nanonets","url":""},"nanonets-ocr2-3b":{"country":"US","id":"nanonets-ocr2-3b","launched":"","modalities":["visual"],"model":"Nanonets OCR2 3B","model_family":"Nanonets OCR","model_variant":"OCR2 3B","model_version":"Nanonets OCR2","name":"Nanonets OCR2 3B","open_source":false,"parameter_size":"3B","provider":"nanonets","url":""},"nemotron-1.5b":{"country":"US","id":"nemotron-1.5b","launched":"2025-07-16","modalities":["text"],"model":"Nemotron 1.5B","model_family":"Nemotron","model_variant":"1.5B","model_version":"Nemotron","name":"Nemotron 1.5B","open_source":true,"parameter_size":"1.5B","provider":"nvidia","url":""},"nemotron-3-nano-30b-a3b":{"country":"US","id":"nemotron-3-nano-30b-a3b","launched":"2025-12-15","modalities":["text"],"model":"Nemotron-3 Nano 30B A3B","model_family":"Nemotron","model_variant":"3-Nano-30B-A3B","model_version":"Nemotron 3","name":"Nemotron-3-Nano-30B-A3B","open_source":"plus","parameter_size":"30B","provider":"nvidia","url":""},"nemotron-3-nano-30b-a3b-base":{"country":"US","id":"nemotron-3-nano-30b-a3b-base","launched":"2025-12-15","modalities":["text"],"model":"Nemotron-3 Nano 30B A3B Base","model_family":"Nemotron","model_variant":"3-Nano-30B-A3B-Base","model_version":"Nemotron 3","name":"Nemotron-3-Nano-30B-A3B-Base","open_source":"plus","parameter_size":"30B","provider":"nvidia","url":""},"nemotron-32b":{"country":"US","id":"nemotron-32b","launched":"2025-06-15","modalities":["text"],"model":"Nemotron 32B","model_family":"Nemotron","model_variant":"Nemotron","model_version":"Nemotron","name":"Nemotron 32B","open_source":true,"parameter_size":"32B","provider":"nvidia","url":""},"nemotron-4-340b-instruct":{"country":"US","id":"nemotron-4-340b-instruct","launched":"","modalities":["text"],"model":"Nemotron-4 340B","model_family":"Nemotron","model_variant":"Nemotron-4 340B","model_version":"Nemotron-4","name":"Nemotron-4 340B Instruct","open_source":true,"parameter_size":"340B","provider":"nvidia","url":""},"nemotron-cascade-14b-thinking":{"country":"US","id":"nemotron-cascade-14b-thinking","launched":"2025-12-16","modalities":["text"],"model":"Nemotron Cascade 14B","model_family":"Nemotron Cascade","model_variant":"Cascade 14B Thinking","model_version":"Nemotron Cascade","name":"Nemotron-Cascade-14B-Thinking","open_source":"plus","parameter_size":"14B","provider":"nvidia","url":""},"nemotron-cascade-8b":{"country":"US","id":"nemotron-cascade-8b","launched":"2025-12-16","modalities":["text"],"model":"Nemotron Cascade 8B","model_family":"Nemotron Cascade","model_variant":"Cascade 8B","model_version":"Nemotron Cascade","name":"Nemotron-Cascade-8B","open_source":"plus","parameter_size":"8B","provider":"nvidia","url":""},"nemotron-cascade-8b-thinking":{"country":"US","id":"nemotron-cascade-8b-thinking","launched":"2025-12-16","modalities":["text"],"model":"Nemotron Cascade 8B","model_family":"Nemotron Cascade","model_variant":"Cascade 8B Thinking","model_version":"Nemotron Cascade","name":"Nemotron-Cascade-8B-Thinking","open_source":"plus","parameter_size":"8B","provider":"nvidia","url":""},"nemotron-flash-1b":{"country":"US","id":"nemotron-flash-1b","launched":"2025-12-02","modalities":["text"],"model":"Nemotron Flash 1B","model_family":"Nemotron Flash","model_variant":"Flash 1B","model_version":"Nemotron Flash","name":"Nemotron-Flash-1B","open_source":true,"parameter_size":"1B","provider":"nvidia","url":""},"nemotron-flash-3b":{"country":"US","id":"nemotron-flash-3b","launched":"2025-12-02","modalities":["text"],"model":"Nemotron Flash 3B","model_family":"Nemotron Flash","model_variant":"Flash 3B","model_version":"Nemotron Flash","name":"Nemotron-Flash-3B","open_source":true,"parameter_size":"3B","provider":"nvidia","url":""},"nemotron-flash-3b-tp":{"country":"US","id":"nemotron-flash-3b-tp","launched":"2025-12-02","modalities":["text"],"model":"Nemotron Flash 3B TP","model_family":"Nemotron Flash","model_variant":"Flash 3B TP","model_version":"Nemotron Flash","name":"Nemotron-Flash-3B-TP","open_source":true,"parameter_size":"3B","provider":"nvidia","url":""},"nemotron-h-47b-reasoning":{"country":"US","id":"nemotron-h-47b-reasoning","launched":"","modalities":["text"],"model":"Nemotron-H 47B","model_family":"Nemotron-H","model_variant":"47B Reasoning","model_version":"Nemotron-H","name":"Nemotron-H-47B-Reasoning","open_source":true,"parameter_size":"47B","provider":"nvidia","url":""},"nemotron-nano-12b-v2-base":{"country":"US","id":"nemotron-nano-12b-v2-base","launched":"2025-10-28","modalities":["text"],"model":"Nemotron Nano 12B","model_family":"Nemotron","model_variant":"Nano 12B v2 Base","model_version":"Nemotron Nano v2","name":"Nemotron Nano 12B v2 Base","open_source":true,"parameter_size":"12B","provider":"nvidia","url":""},"nemotron-nano-8b-v1":{"country":"US","id":"nemotron-nano-8b-v1","launched":"","modalities":["text"],"model":"Nemotron Nano 8B V1","model_family":"Nemotron","model_variant":"Nemotron Nano","model_version":"Nemotron Nano V1","name":"Nemotron Nano 8B V1","open_source":true,"parameter_size":"8B","provider":"nvidia","url":""},"nemotron-nano-v2-9b":{"country":"US","id":"nemotron-nano-v2-9b","launched":"","modalities":["text"],"model":"Nemotron-Nano-v2 9B","model_family":"Nemotron","model_variant":"Nemotron Nano","model_version":"Nemotron Nano V2","name":"Nemotron-Nano-v2 9B","open_source":true,"parameter_size":"9B","provider":"nvidia","url":""},"nougat":{"country":"US","id":"nougat","launched":"","modalities":["visual"],"model":"Nougat","model_family":"Nougat","model_variant":"Base","model_version":"Nougat","name":"Nougat","open_source":true,"parameter_size":"","provider":"meta","url":""},"nous-hermes-13b":{"country":"US","id":"nous-hermes-13b","launched":"","modalities":["text"],"model":"Nous Hermes 13B","model_family":"Nous Hermes","model_variant":"13B","model_version":"Nous Hermes","name":"Nous Hermes 13B","open_source":true,"parameter_size":"13B","provider":"nousresearch","url":""},"nous-hermes-2-mixtral-8x7b-dpo":{"active_parameters":"13B","country":"US","id":"nous-hermes-2-mixtral-8x7b-dpo","launched":"","modalities":["text"],"model":"Nous Hermes 2 Mixtral","model_family":"Nous Hermes","model_variant":"Nous Hermes 2 Mixtral 8x7B","model_version":"Nous Hermes 2","name":"Nous Hermes 2 Mixtral 8x7B DPO","open_source":true,"parameter_size":"47B","provider":"nousresearch","url":""},"nouscoder-14b":{"country":"US","id":"nouscoder-14b","launched":"2026-01-07","modalities":["text"],"model":"NousCoder 14B","model_family":"NousCoder","model_variant":"14B","model_version":"NousCoder","name":"NousCoder-14B","open_source":true,"parameter_size":"14B","provider":"nousresearch","url":""},"nova-lite":{"country":"US","id":"nova-lite","launched":"2025-12-01","modalities":["text"],"model":"Nova Lite","model_family":"Nova","model_variant":"Nova Lite","model_version":"Nova","name":"Nova Lite","open_source":false,"parameter_size":"","provider":"amazon","url":""},"nova-micro":{"country":"US","id":"nova-micro","launched":"2025-12-01","modalities":["text"],"model":"Nova Micro","model_family":"Nova","model_variant":"Nova Micro","model_version":"Nova","name":"Nova Micro","open_source":false,"parameter_size":"","provider":"amazon","url":""},"nova-pro":{"country":"US","id":"nova-pro","launched":"2025-12-01","modalities":["text"],"model":"Nova Pro","model_family":"Nova","model_variant":"Nova Pro","model_version":"Nova","name":"Nova Pro","open_source":false,"parameter_size":"","provider":"amazon","url":""},"nvidia-llama-3.3-nemotron-super-49b-v1.5":{"country":"US","id":"nvidia-llama-3.3-nemotron-super-49b-v1.5","launched":"2024-12-07","modalities":["text"],"model":"Llama 3.3 Nemotron","model_family":"Llama","model_variant":"Llama 3.3 Nemotron Super 49B","model_version":"Llama 3.3","name":"NVIDIA Llama 3.3 Nemotron Super 49B v1.5","open_source":true,"parameter_size":"49B","provider":"nvidia","url":""},"nvidia-nemotron-3-nano-30b-a3b-bf16":{"country":"US","id":"nvidia-nemotron-3-nano-30b-a3b-bf16","launched":"2025-12-15","modalities":["text"],"model":"Nemotron-3 Nano 30B A3B BF16","model_family":"Nemotron","model_variant":"3-Nano-30B-A3B-BF16","model_version":"Nemotron 3","name":"NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","open_source":"plus","parameter_size":"30B","provider":"nvidia","url":""},"nvidia-nemotron-nano-9b-v2":{"country":"US","id":"nvidia-nemotron-nano-9b-v2","launched":"2025-08-12","modalities":["text"],"model":"NVIDIA Nemotron Nano 9B","model_family":"Nemotron","model_variant":"Nemotron Nano 9B v2","model_version":"Nemotron Nano v2","name":"NVIDIA-Nemotron-Nano-9B-v2","open_source":true,"parameter_size":"9B","provider":"nvidia","url":""},"nvila-internal-15b-v1":{"country":"US","id":"nvila-internal-15b-v1","launched":"","modalities":["visual"],"model":"NVILA 15B","model_family":"NVILA","model_variant":"Internal 15B v1","model_version":"NVILA","name":"NVILA Internal 15B v1","open_source":false,"parameter_size":"15B","provider":"nvidia","url":""},"nxcode-cq-7b-orpo":{"country":"US","id":"nxcode-cq-7b-orpo","launched":"","modalities":["text"],"model":"Nxcode CQ 7B Orpo","model_family":"Nxcode","model_variant":"Nxcode CQ 7B Orpo","model_version":"Nxcode","name":"Nxcode CQ 7B Orpo","open_source":true,"parameter_size":"7B","provider":"nexusflow","url":""},"o1":{"country":"US","id":"o1","launched":"2024-12-05","modalities":["text"],"model":"o1","model_family":"o1","model_variant":"o1","model_version":"o1","name":"o1","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"o1-2024-12-17":{"country":"US","id":"o1-2024-12-17","launched":"2024-12-05","modalities":["text"],"model":"o1","model_family":"o1","model_variant":"o1 20241217","model_version":"o1","name":"o1 (2024-12-17)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o1-2024-12-17-high":{"country":"US","id":"o1-2024-12-17-high","launched":"2024-12-05","modalities":["text"],"model":"o1","model_family":"o1","model_variant":"o1 2024-12-17 High","model_version":"o1","name":"o1 (2024-12-17 High)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o1-mini":{"country":"US","id":"o1-mini","launched":"2024-12-05","modalities":["text"],"model":"o1 Mini","model_family":"o1","model_variant":"o1 Mini","model_version":"o1","name":"o1 Mini","open_source":false,"parameter_size":"100Be","provider":"openai","url":""},"o1-mini-2024-09-12":{"country":"US","id":"o1-mini-2024-09-12","launched":"2024-12-05","modalities":["text"],"model":"o1 Mini","model_family":"o1","model_variant":"o1 Mini 20240912","model_version":"o1","name":"o1 Mini (2024-09-12)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o1-mini-2024-09-12-high":{"country":"US","id":"o1-mini-2024-09-12-high","launched":"2024-12-05","modalities":["text"],"model":"o1 Mini High","model_family":"o1","model_variant":"o1 Mini 2024-09-12 High","model_version":"o1","name":"o1 Mini (2024-09-12 High)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o1-preview":{"country":"US","id":"o1-preview","launched":"2024-12-05","modalities":["text"],"model":"o1","model_family":"o1","model_variant":"o1-preview","model_version":"o1","name":"o1-preview","open_source":false,"parameter_size":"100Be","provider":"openai","url":""},"o1-preview-2024-09-12":{"country":"US","id":"o1-preview-2024-09-12","launched":"2024-12-05","modalities":["text"],"model":"o1","model_family":"o1","model_variant":"o1 Preview 2024 09 12","model_version":"o1","name":"o1-Preview-2024-09-12","open_source":false,"parameter_size":"","provider":"openai","url":""},"o1-pro":{"country":"US","id":"o1-pro","launched":"2024-12-05","modalities":["text"],"model":"o1 Pro","model_family":"o1","model_variant":"o1 Pro","model_version":"o1","name":"o1-Pro","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3":{"country":"US","id":"o3","launched":"2025-04-16","modalities":["text","visual"],"model":"o3","model_family":"o3","model_variant":"o3","model_version":"o3","name":"o3","open_source":false,"parameter_size":"500Be","provider":"openai","url":""},"o3-2025-04-16":{"country":"US","id":"o3-2025-04-16","launched":"2025-06-10","modalities":["text"],"model":"o3","model_family":"o3","model_variant":"o3 20250416","model_version":"o3","name":"o3 (2025-04-16)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-high":{"country":"US","id":"o3-high","launched":"2025-06-10","modalities":["visual"],"model":"o3","model_family":"o3","model_variant":"o3","model_version":"o3","name":"o3-high","open_source":false,"parameter_size":"1000Be","provider":"openai","url":""},"o3-high-plus-gpt-4.1":{"country":"US","id":"o3-high-plus-gpt-4.1","launched":"","modalities":["text"],"model":"o3 High + GPT-4.1","model_family":"Composite","model_variant":"o3 High + GPT-4.1","model_version":"Composite","name":"o3 (High) + GPT-4.1","open_source":false,"parameter_size":"","provider":"ensemble","url":""},"o3-low":{"country":"US","id":"o3-low","launched":"2025-06-10","modalities":["text"],"model":"o3","model_family":"o3","model_variant":"o3 Low","model_version":"o3","name":"o3 (Low)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-medium":{"country":"US","id":"o3-medium","launched":"2025-06-10","modalities":["text"],"model":"o3","model_family":"o3","model_variant":"o3 Medium","model_version":"o3","name":"o3 Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-mini":{"country":"US","id":"o3-mini","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 mini","model_version":"o3","name":"o3 mini","open_source":false,"parameter_size":"200Be","provider":"openai","url":""},"o3-mini-2025-01-31":{"country":"US","id":"o3-mini-2025-01-31","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 Mini 20250131","model_version":"o3","name":"o3 Mini (2025-01-31)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-mini-2025-01-31-high":{"country":"US","id":"o3-mini-2025-01-31-high","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 mini 2025-01-31 High","model_version":"o3","name":"o3 mini 2025-01-31 (High)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-mini-2025-01-31-low":{"country":"US","id":"o3-mini-2025-01-31-low","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 mini 2025-01-31 Low","model_version":"o3","name":"o3 mini 2025-01-31 (Low)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-mini-2025-01-31-med":{"country":"US","id":"o3-mini-2025-01-31-med","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 mini 2025-01-31 Med","model_version":"o3","name":"o3 mini 2025-01-31 (Med)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-mini-high":{"country":"US","id":"o3-mini-high","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 mini-high","model_version":"o3","name":"o3 mini-high","open_source":false,"parameter_size":"300Be","provider":"openai","url":""},"o3-mini-low":{"country":"US","id":"o3-mini-low","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 mini Low","model_version":"o3","name":"o3 mini (Low)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-mini-medium":{"country":"US","id":"o3-mini-medium","launched":"2025-01-31","modalities":["text"],"model":"o3 Mini","model_family":"o3","model_variant":"o3 mini Medium","model_version":"o3","name":"o3 mini (Medium)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-preview-low":{"country":"US","id":"o3-preview-low","launched":"2025-06-10","modalities":["text"],"model":"o3 Preview","model_family":"o3","model_variant":"o3 Preview Low","model_version":"o3","name":"o3 Preview (Low)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-pro":{"country":"US","id":"o3-pro","launched":"2025-06-10","modalities":["text"],"model":"o3 Pro","model_family":"o3","model_variant":"o3-pro","model_version":"o3","name":"o3-pro","open_source":false,"parameter_size":"1500Be","provider":"openai","url":""},"o3-pro-2025-06-10-high":{"country":"US","id":"o3-pro-2025-06-10-high","launched":"2025-06-10","modalities":["text"],"model":"o3 Pro","model_family":"o3","model_variant":"o3 Pro High 20250610","model_version":"o3","name":"o3 Pro High (2025-06-10)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-pro-high":{"country":"US","id":"o3-pro-high","launched":"2025-06-10","modalities":["text"],"model":"o3 Pro","model_family":"o3","model_variant":"o3 Pro High","model_version":"o3","name":"o3 Pro High","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-pro-low":{"country":"US","id":"o3-pro-low","launched":"2025-06-10","modalities":["text"],"model":"o3 Pro","model_family":"o3","model_variant":"o3 Pro Low","model_version":"o3","name":"o3 Pro (Low)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o3-pro-medium":{"country":"US","id":"o3-pro-medium","launched":"2025-06-10","modalities":["text"],"model":"o3 Pro","model_family":"o3","model_variant":"o3 Pro Medium","model_version":"o3","name":"o3 Pro (Medium)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o4-mini":{"country":"US","id":"o4-mini","launched":"2025-04-16","modalities":["text"],"model":"o4 Mini","model_family":"o4","model_variant":"o4 mini","model_version":"o4","name":"o4 mini","open_source":false,"parameter_size":"300Be","provider":"openai","url":""},"o4-mini-2025-04-16":{"country":"US","id":"o4-mini-2025-04-16","launched":"2025-04-16","modalities":["text"],"model":"o4 Mini","model_family":"o4","model_variant":"O4 Mini 20250416","model_version":"o4","name":"O4 Mini (2025-04-16)","open_source":false,"parameter_size":"","provider":"openai","url":""},"o4-mini-2025-04-16-high":{"country":"US","id":"o4-mini-2025-04-16-high","launched":"2025-04-16","modalities":["text"],"model":"o4 Mini","model_family":"o4","model_variant":"o4 Mini High","model_version":"o4","name":"o4-mini (2025-04-16) High","open_source":false,"parameter_size":"","provider":"openai","url":""},"o4-mini-high":{"country":"US","id":"o4-mini-high","launched":"2025-04-16","modalities":["text"],"model":"o4 Mini","model_family":"o4","model_variant":"o4-Mini High","model_version":"o4","name":"o4-Mini High","open_source":false,"parameter_size":"","provider":"openai","url":""},"o4-mini-low":{"country":"US","id":"o4-mini-low","launched":"2025-04-16","modalities":["text"],"model":"o4 Mini","model_family":"o4","model_variant":"o4-Mini Low","model_version":"o4","name":"o4-Mini Low","open_source":false,"parameter_size":"","provider":"openai","url":""},"o4-mini-medium":{"country":"US","id":"o4-mini-medium","launched":"2025-04-16","modalities":["text"],"model":"o4 Mini","model_family":"o4","model_variant":"o4-Mini Medium","model_version":"o4","name":"o4-Mini Medium","open_source":false,"parameter_size":"","provider":"openai","url":""},"oasst-pythia-12b":{"country":"US","id":"oasst-pythia-12b","launched":"","modalities":["text"],"model":"OASST Pythia","model_family":"OASST","model_variant":"Pythia-12B","model_version":"OASST-Pythia","name":"OASST-Pythia-12B","open_source":true,"parameter_size":"12B","provider":"oasst","url":""},"ocrflux":{"country":"CN","id":"ocrflux","launched":"","modalities":["visual"],"model":"OCRFlux","model_family":"OCRFlux","model_variant":"Base","model_version":"OCRFlux","name":"OCRFlux","open_source":true,"parameter_size":"","provider":"ocrflux","url":""},"ocrflux-3b":{"country":"CN","id":"ocrflux-3b","launched":"","modalities":["visual"],"model":"OCRFlux-3B","model_family":"OCRFlux","model_variant":"3B","model_version":"OCRFlux","name":"OCRFlux-3B","open_source":true,"parameter_size":"3B","provider":"chatdoc","url":""},"ocrverse":{"country":"CN","id":"ocrverse","launched":"","modalities":["visual"],"model":"OCRVerse","model_family":"OCRVerse","model_variant":"Base","model_version":"OCRVerse","name":"OCRVerse","open_source":true,"parameter_size":"","provider":"ocrverse","url":""},"olmo-2-0325-32b-instruct":{"country":"US","id":"olmo-2-0325-32b-instruct","launched":"2025-03-14","modalities":["text"],"model":"OLMo 2 32B","model_family":"OLMo","model_variant":"OLMo2-32B","model_version":"OLMo 2","name":"OLMo-2-0325-32B-Instruct","open_source":"plus","parameter_size":"32B","provider":"allenai","url":""},"olmo-2-0425-1b-base":{"country":"US","id":"olmo-2-0425-1b-base","launched":"2025-05-01","modalities":["text"],"model":"OLMo 2 1B","model_family":"OLMo","model_variant":"OLMo2-0425-1B-Base","model_version":"OLMo 2","name":"OLMo-2-0425-1B-Base","open_source":"plus","parameter_size":"1B","provider":"allenai","url":""},"olmo-2-0425-1b-instruct":{"country":"US","id":"olmo-2-0425-1b-instruct","launched":"2025-05-01","modalities":["text"],"model":"OLMo 2 1B","model_family":"OLMo","model_variant":"OLMo2-0425-1B-Instruct","model_version":"OLMo 2","name":"OLMo-2-0425-1B-Instruct","open_source":"plus","parameter_size":"1B","provider":"allenai","url":""},"olmo-2-32b-base":{"country":"US","id":"olmo-2-32b-base","launched":"2025-03-14","modalities":["text"],"model":"OLMo 2 32B","model_family":"OLMo","model_variant":"OLMo2-32B Base","model_version":"OLMo 2","name":"OLMo-2-32B-Base","open_source":"plus","parameter_size":"32B","provider":"allenai","url":""},"olmo-3-7b":{"country":"US","id":"olmo-3-7b","launched":"2025-11-20","modalities":["text"],"model":"OLMo 3 7B","model_family":"OLMo","model_variant":"OLMo 3 7B","model_version":"OLMo 3","name":"OLMo 3 7B","open_source":"plus","parameter_size":"7B","provider":"allenai","url":""},"olmo-3-7b-sft":{"country":"US","id":"olmo-3-7b-sft","launched":"2025-11-24","modalities":["text"],"model":"OLMo 3 7B","model_family":"OLMo","model_variant":"OLMo 3 7B SFT","model_version":"OLMo 3","name":"OLMo 3 7B SFT","open_source":"plus","parameter_size":"7B","provider":"allenai","url":""},"olmo-3-think-32b":{"country":"US","id":"olmo-3-think-32b","launched":"2025-11-20","modalities":["text"],"model":"OLMo 3 Think 32B","model_family":"OLMo","model_variant":"OLMo 3 Think 32B","model_version":"OLMo 3","name":"OLMo-3-Think-32B","open_source":"plus","parameter_size":"32B","provider":"allenai","url":""},"olmo-3-think-32b-dpo":{"country":"US","id":"olmo-3-think-32b-dpo","launched":"2025-11-20","modalities":["text"],"model":"OLMo 3 Think 32B DPO","model_family":"OLMo","model_variant":"OLMo 3 Think 32B DPO","model_version":"OLMo 3","name":"OLMo-3-Think-32B-DPO","open_source":"plus","parameter_size":"32B","provider":"allenai","url":""},"olmo-3-think-32b-sft":{"country":"US","id":"olmo-3-think-32b-sft","launched":"2025-11-20","modalities":["text"],"model":"OLMo 3 Think 32B SFT","model_family":"OLMo","model_variant":"OLMo 3 Think 32B SFT","model_version":"OLMo 3","name":"OLMo-3-Think-32B-SFT","open_source":"plus","parameter_size":"32B","provider":"allenai","url":""},"olmo-3.1-think-32b":{"country":"US","id":"olmo-3.1-think-32b","launched":"2025-12-10","modalities":["text"],"model":"OLMo 3.1 Think 32B","model_family":"OLMo","model_variant":"OLMo 3.1 Think 32B","model_version":"OLMo 3.1","name":"OLMo-3.1-Think-32B","open_source":"plus","parameter_size":"32B","provider":"allenai","url":""},"olmo-7b-instruct":{"country":"US","id":"olmo-7b-instruct","launched":"","modalities":["text"],"model":"OLMo 7B","model_family":"OLMo","model_variant":"OLMo 7B Instruct","model_version":"OLMo","name":"OLMo 7B Instruct","open_source":"plus","parameter_size":"7B","provider":"allenai","url":""},"olmocr":{"country":"US","id":"olmocr","launched":"","modalities":["visual"],"model":"OLMOCR","model_family":"OLMOCR","model_variant":"Base","model_version":"OLMOCR","name":"OLMOCR","open_source":true,"parameter_size":"","provider":"allenai","url":""},"olmocr-2-7b-1025":{"country":"US","id":"olmocr-2-7b-1025","launched":"2025-10-22","modalities":["visual"],"model":"OLMOCR 2 7B","model_family":"OLMOCR","model_variant":"2-7B-1025","model_version":"OLMOCR 2","name":"OLMOCR-2-7B-1025","open_source":true,"parameter_size":"7B","provider":"allenai","url":""},"open-mixtral-8x22b-2404":{"country":"FR","id":"open-mixtral-8x22b-2404","launched":"2024-04-01","modalities":["text"],"model":"Open Mixtral 8x22B","model_family":"Mixtral","model_variant":"Open Mixtral 8x22B 24.04","model_version":"Mixtral","name":"Open Mixtral 8x22B (24.04)","open_source":true,"parameter_size":"176B","provider":"mistral","url":""},"openai-deepresearch":{"country":"US","id":"openai-deepresearch","launched":"2025-02-02","modalities":["text"],"model":"OpenAI DeepResearch","model_family":"DeepResearch","model_variant":"OpenAI DeepResearch","model_version":"DeepResearch","name":"OpenAI DeepResearch","open_source":false,"parameter_size":"","provider":"openai","url":""},"openchat-3-6-8b-20240522":{"country":"US","id":"openchat-3-6-8b-20240522","launched":"","modalities":["text"],"model":"Openchat 3 6.8B 20240522","model_family":"OpenChat-3.5-7B-0106","model_variant":"Openchat 3 6.8B 20240522","model_version":"OpenChat-3.5-7B-0106","name":"Openchat 3 6.8B 20240522","open_source":true,"parameter_size":"6.8B","provider":"openchat","url":""},"openchat-3.5":{"country":"US","id":"openchat-3.5","launched":"","modalities":["text"],"model":"OpenChat","model_family":"OpenChat","model_variant":"OpenChat 3.5","model_version":"OpenChat 3.5","name":"OpenChat 3.5","open_source":true,"parameter_size":"","provider":"openchat","url":""},"openchat-3.5-0106":{"country":"US","id":"openchat-3.5-0106","launched":"","modalities":["text"],"model":"OpenChat","model_family":"OpenChat","model_variant":"OpenChat 3.5 0106","model_version":"OpenChat 3.5 0106","name":"OpenChat 3.5 0106","open_source":true,"parameter_size":"","provider":"openchat","url":""},"openchat-3.5-0106-gemma":{"country":"US","id":"openchat-3.5-0106-gemma","launched":"","modalities":["text"],"model":"OpenChat","model_family":"OpenChat","model_variant":"OpenChat 3.5 0106 Gemma","model_version":"OpenChat 3.5 0106","name":"OpenChat 3.5 0106 Gemma","open_source":true,"parameter_size":"","provider":"openchat","url":""},"openchat-3.5-7b-0106":{"country":"US","id":"openchat-3.5-7b-0106","launched":"","modalities":["text"],"model":"OpenChat-3.5-7B-0106","model_family":"OpenChat-3.5-7B-0106","model_variant":"OpenChat-3.5-7B-0106","model_version":"OpenChat-3.5-7B-0106","name":"OpenChat-3.5-7B-0106","open_source":true,"parameter_size":"7B","provider":"openchat","url":""},"openchat-3.6-8b-20240522":{"country":"US","id":"openchat-3.6-8b-20240522","launched":"","modalities":["text"],"model":"Openchat 3.6 8B 20240522","model_family":"OpenChat-3.5-7B-0106","model_variant":"Openchat 3.6 8B 20240522","model_version":"OpenChat-3.5-7B-0106","name":"Openchat 3.6 8B 20240522","open_source":true,"parameter_size":"8B","provider":"openchat","url":""},"opencodeinterpreter-ds-1.3b":{"country":"Global","id":"opencodeinterpreter-ds-1.3b","launched":"","modalities":["text"],"model":"Opencodeinterpreter DS 1.3B","model_family":"OpenCodeInterpreter","model_variant":"Opencodeinterpreter DS 1.3B","model_version":"OpenCodeInterpreter DS","name":"Opencodeinterpreter DS 1.3B","open_source":true,"parameter_size":"1.3B","provider":"map","url":""},"opencodeinterpreter-ds-33b":{"country":"Global","id":"opencodeinterpreter-ds-33b","launched":"","modalities":["text"],"model":"OpenCodeInterpreter DS 33B","model_family":"OpenCodeInterpreter","model_variant":"OpenCodeInterpreter-DS-33B","model_version":"OpenCodeInterpreter DS","name":"OpenCodeInterpreter-DS-33B","open_source":true,"parameter_size":"33B","provider":"map","url":""},"opencodeinterpreter-ds-6.7b":{"country":"Global","id":"opencodeinterpreter-ds-6.7b","launched":"","modalities":["text"],"model":"OpenCodeInterpreter DS 6.7B","model_family":"OpenCodeInterpreter","model_variant":"OpenCodeInterpreter-DS-6.7B","model_version":"OpenCodeInterpreter DS","name":"OpenCodeInterpreter-DS-6.7B","open_source":true,"parameter_size":"6.7B","provider":"map","url":""},"opencoder-1-5b-instruct":{"country":"US","id":"opencoder-1-5b-instruct","launched":"","modalities":["text"],"model":"Opencoder 1.5B","model_family":"OpenCoder","model_variant":"Opencoder 1.5B Instruct","model_version":"OpenCoder-8B-Instruct","name":"Opencoder 1.5B Instruct","open_source":true,"parameter_size":"1.5B","provider":"map","url":""},"opencoder-1.5b-instruct":{"country":"US","id":"opencoder-1.5b-instruct","launched":"","modalities":["text"],"model":"Opencoder 1.5B","model_family":"OpenCoder","model_variant":"Opencoder 1.5B Instruct","model_version":"OpenCoder-8B-Instruct","name":"Opencoder 1.5B Instruct","open_source":true,"parameter_size":"1.5B","provider":"map","url":""},"opencoder-8b-instruct":{"country":"US","id":"opencoder-8b-instruct","launched":"","modalities":["text"],"model":"OpenCoder 8B","model_family":"OpenCoder","model_variant":"OpenCoder-8B-Instruct","model_version":"OpenCoder","name":"OpenCoder-8B-Instruct","open_source":true,"parameter_size":"8B","provider":"map","url":""},"opencodereasoning-nemotron-1.1-32b":{"country":"US","id":"opencodereasoning-nemotron-1.1-32b","launched":"2025-07-16","modalities":["text"],"model":"OpenCodeReasoning Nemotron 32B","model_family":"Nemotron","model_variant":"OpenCodeReasoning","model_version":"Nemotron OpenCodeReasoning","name":"OpenCodeReasoning-Nemotron-1.1-32B","open_source":true,"parameter_size":"32B","provider":"nvidia","url":""},"openhands-lm-32b-v0.1":{"country":"US","id":"openhands-lm-32b-v0.1","launched":"","modalities":["text"],"model":"OpenHands LM 32B","model_family":"OpenHands","model_variant":"LM 32B v0.1","model_version":"OpenHands","name":"OpenHands LM 32B v0.1","open_source":true,"parameter_size":"32B","provider":"openhands","url":""},"openhermes-2.5-code-290k-13b":{"country":"US","id":"openhermes-2.5-code-290k-13b","launched":"","modalities":["text"],"model":"OpenHermes-2.5-Code-290k-13B","model_family":"OpenHermes-2.5-Code-290k-13B","model_variant":"OpenHermes-2.5-Code-290k-13B","model_version":"OpenHermes-2.5-Code-290k-13B","name":"OpenHermes-2.5-Code-290k-13B","open_source":true,"parameter_size":"13B","provider":"openhermes","url":""},"openhermes-2.5-mistral-7b":{"country":"US","id":"openhermes-2.5-mistral-7b","launched":"","modalities":["text"],"model":"OpenHermes 2.5 Mistral","model_family":"OpenHermes","model_variant":"2.5 Mistral 7B","model_version":"OpenHermes 2.5","name":"OpenHermes-2.5-Mistral-7B","open_source":true,"parameter_size":"7B","provider":"openhermes","url":""},"openpangu-r-72b-2512-fast-thinking":{"country":"CN","id":"openpangu-r-72b-2512-fast-thinking","launched":"2026-01-05","modalities":["text"],"model":"openPangu-R 72B","model_family":"openPangu-R","model_variant":"72B-2512 Fast Thinking","model_version":"openPangu-R","name":"openPangu-R-72B-2512 Fast Thinking","open_source":true,"parameter_size":"72B","provider":"freedomintelligence","url":""},"openpangu-r-72b-2512-slow-thinking":{"country":"CN","id":"openpangu-r-72b-2512-slow-thinking","launched":"2026-01-05","modalities":["text"],"model":"openPangu-R 72B","model_family":"openPangu-R","model_variant":"72B-2512 Slow Thinking","model_version":"openPangu-R","name":"openPangu-R-72B-2512 Slow Thinking","open_source":true,"parameter_size":"72B","provider":"freedomintelligence","url":""},"openreasoning-nemotron-1.5b":{"country":"US","id":"openreasoning-nemotron-1.5b","launched":"2025-07-16","modalities":["text"],"model":"OpenReasoning Nemotron 1.5B","model_family":"Nemotron","model_variant":"OpenReasoning","model_version":"Nemotron OpenReasoning","name":"OpenReasoning-Nemotron-1.5B","open_source":true,"parameter_size":"1.5B","provider":"nvidia","url":""},"openreasoning-nemotron-14b":{"country":"US","id":"openreasoning-nemotron-14b","launched":"2025-07-16","modalities":["text"],"model":"OpenReasoning Nemotron 14B","model_family":"Nemotron","model_variant":"OpenReasoning","model_version":"Nemotron OpenReasoning","name":"OpenReasoning-Nemotron-14B","open_source":true,"parameter_size":"14B","provider":"nvidia","url":""},"openreasoning-nemotron-32b":{"country":"US","id":"openreasoning-nemotron-32b","launched":"2025-07-16","modalities":["text"],"model":"OpenReasoning Nemotron 32B","model_family":"Nemotron","model_variant":"OpenReasoning","model_version":"Nemotron OpenReasoning","name":"OpenReasoning-Nemotron-32B","open_source":true,"parameter_size":"32B","provider":"nvidia","url":""},"openreasoning-nemotron-7b":{"country":"US","id":"openreasoning-nemotron-7b","launched":"2025-07-16","modalities":["text"],"model":"OpenReasoning Nemotron 7B","model_family":"Nemotron","model_variant":"OpenReasoning","model_version":"Nemotron OpenReasoning","name":"OpenReasoning-Nemotron-7B","open_source":true,"parameter_size":"7B","provider":"nvidia","url":""},"openthinker2-32b":{"country":"CN","id":"openthinker2-32b","launched":"","modalities":["text"],"model":"OpenThinker2 32B","model_family":"OpenThinker","model_variant":"OpenThinker2 32B","model_version":"OpenThinker2","name":"OpenThinker2 32B","open_source":true,"parameter_size":"32B","provider":"openthinker","url":""},"opt-175b":{"country":"US","id":"opt-175b","launched":"2022-05-12","modalities":["text"],"model":"OPT","model_family":"OPT","model_variant":"175B","model_version":"175B","name":"OPT-175B","open_source":"plus","parameter_size":"175B","provider":"meta","url":"https://huggingface.co/facebook/opt-175b"},"optimus-alpha":{"country":"CN","id":"optimus-alpha","launched":"","modalities":["text"],"model":"Optimus Alpha","model_family":"Optimus","model_variant":"Optimus Alpha","model_version":"Optimus","name":"Optimus Alpha","open_source":true,"parameter_size":"","provider":"openbmb","url":""},"ouro-1.4b":{"country":"CN","id":"ouro-1.4b","launched":"2025-10-28","modalities":["text"],"model":"Ouro 1.4B","model_family":"Ouro","model_variant":"1.4B","model_version":"Ouro","name":"Ouro-1.4B","open_source":true,"parameter_size":"1.4B","provider":"bytedance","url":""},"ouro-2.6b":{"country":"CN","id":"ouro-2.6b","launched":"2025-10-28","modalities":["text"],"model":"Ouro 2.6B","model_family":"Ouro","model_variant":"2.6B","model_version":"Ouro","name":"Ouro-2.6B","open_source":true,"parameter_size":"2.6B","provider":"bytedance","url":""},"ovis2.6-30b-a3b":{"active_parameters":"3B","country":"CN","id":"ovis2.6-30b-a3b","launched":"2026-02-12","modalities":["text","visual"],"model":"Ovis2.6 30B A3B","model_family":"Ovis","model_variant":"Ovis2.6-30B-A3B","model_version":"Ovis 2.6","name":"Ovis2.6-30B-A3B","open_source":true,"parameter_size":"30B","provider":"alibaba","url":"https://huggingface.co/AIDC-AI/Ovis2.6-30B-A3B"},"paddleocr-vl":{"country":"CN","id":"paddleocr-vl","launched":"","modalities":["visual"],"model":"PaddleOCR-VL","model_family":"PaddleOCR","model_variant":"VL","model_version":"PaddleOCR","name":"PaddleOCR-VL","open_source":true,"parameter_size":"","provider":"paddlepaddle","url":""},"paligemma-mix-3b":{"country":"US","id":"paligemma-mix-3b","launched":"2024-05-24","modalities":["visual"],"model":"PaliGemma-mix-3B","model_family":"PaliGemma","model_variant":"mix-3B","model_version":"PaliGemma","name":"PaliGemma-mix-3B","open_source":true,"parameter_size":"3B","provider":"google","url":""},"palm-2":{"country":"US","id":"palm-2","launched":"","modalities":["text"],"model":"PaLM 2","model_family":"PaLM","model_variant":"PaLM 2","model_version":"PaLM 2","name":"PaLM 2","open_source":false,"parameter_size":"","provider":"google","url":""},"palm-540b":{"country":"US","id":"palm-540b","launched":"2022-04-04","modalities":["text"],"model":"PaLM 540B","model_family":"PaLM","model_variant":"PaLM 540B","model_version":"PaLM","name":"PaLM 540B","open_source":false,"parameter_size":"540B","provider":"google","url":""},"palm-62b":{"country":"US","id":"palm-62b","launched":"2022-04-04","modalities":["text"],"model":"PaLM 62B","model_family":"PaLM","model_variant":"PaLM 62B","model_version":"PaLM","name":"PaLM 62B","open_source":false,"parameter_size":"62B","provider":"google","url":""},"palm-cont-62b":{"country":"US","id":"palm-cont-62b","launched":"2022-04-04","modalities":["text"],"model":"PaLM-cont 62B","model_family":"PaLM","model_variant":"PaLM-cont 62B","model_version":"PaLM","name":"PaLM-cont 62B","open_source":false,"parameter_size":"62B","provider":"google","url":""},"perplexity-pro":{"country":"US","id":"perplexity-pro","launched":"2024-12-07","modalities":["text"],"model":"Perplexity Pro","model_family":"Perplexity","model_variant":"Pro","model_version":"Perplexity","name":"Perplexity Pro","open_source":false,"parameter_size":"","provider":"perplexity","url":""},"phi-1":{"country":"US","id":"phi-1","launched":"","modalities":["text"],"model":"Phi-1","model_family":"Phi","model_variant":"Phi-1","model_version":"Phi","name":"Phi-1","open_source":true,"parameter_size":"1.3B","provider":"microsoft","url":""},"phi-1.5":{"country":"US","id":"phi-1.5","launched":"","modalities":["text"],"model":"Phi-1.5","model_family":"Phi","model_variant":"Phi-1.5","model_version":"Phi","name":"Phi-1.5","open_source":true,"parameter_size":"1.3B","provider":"microsoft","url":""},"phi-2":{"country":"US","id":"phi-2","launched":"","modalities":["text"],"model":"Phi-2","model_family":"Phi","model_variant":"Phi-2","model_version":"Phi","name":"Phi-2","open_source":true,"parameter_size":"2.7B","provider":"microsoft","url":""},"phi-2-2.7b":{"country":"US","id":"phi-2-2.7b","launched":"","modalities":["text"],"model":"phi-2-2.7B","model_family":"phi-2-2.7B","model_variant":"phi-2-2.7B","model_version":"phi-2-2.7B","name":"phi-2-2.7B","open_source":true,"parameter_size":"2.7B","provider":"microsoft","url":""},"phi-2-dpo":{"country":"US","id":"phi-2-dpo","launched":"","modalities":["text"],"model":"Phi-2 DPO","model_family":"Phi","model_variant":"Phi-2 DPO","model_version":"Phi","name":"Phi-2 DPO","open_source":true,"parameter_size":"2.7B","provider":"microsoft","url":""},"phi-2-sft":{"country":"US","id":"phi-2-sft","launched":"","modalities":["text"],"model":"Phi-2 SFT","model_family":"Phi","model_variant":"Phi-2 SFT","model_version":"Phi","name":"Phi-2 SFT","open_source":true,"parameter_size":"2.7B","provider":"microsoft","url":""},"phi-3-medium-128k-instruct":{"country":"US","id":"phi-3-medium-128k-instruct","launched":"","modalities":["text"],"model":"PHI 3 Medium 128k","model_family":"Phi","model_variant":"PHI 3 Medium 128k Instruct","model_version":"Phi 3.5","name":"PHI 3 Medium 128k Instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3-medium-4k-instruct":{"country":"US","id":"phi-3-medium-4k-instruct","launched":"2024-04-23","modalities":["text"],"model":"Phi-3 Medium 4K Instruct","model_family":"Phi","model_variant":"Phi-3 Medium 4K Instruct","model_version":"Phi 3","name":"Phi-3 Medium 4K Instruct","open_source":true,"parameter_size":"14B","provider":"microsoft","url":""},"phi-3-mini-128k-instruct":{"country":"US","id":"phi-3-mini-128k-instruct","launched":"","modalities":["text"],"model":"PHI 3 Mini 128k","model_family":"Phi","model_variant":"PHI 3 Mini 128k Instruct","model_version":"Phi 3.5","name":"PHI 3 Mini 128k Instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3-mini-4k-instruct":{"country":"US","id":"phi-3-mini-4k-instruct","launched":"","modalities":["text"],"model":"Phi-3-mini-4k-instruct","model_family":"Phi-3-mini-4k-instruct","model_variant":"Phi-3-mini-4k-instruct","model_version":"Phi-3-mini-4k-instruct","name":"Phi-3-mini-4k-instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3-mini-4k-instruct-june-2024":{"country":"US","id":"phi-3-mini-4k-instruct-june-2024","launched":"2024-06-01","modalities":["text"],"model":"Phi-3 Mini","model_family":"Phi","model_variant":"Phi-3 Mini 4K","model_version":"Phi-3","name":"Phi-3 Mini 4K Instruct (June 2024)","open_source":true,"parameter_size":"3.8B","provider":"microsoft","url":""},"phi-3-small-128k-instruct":{"country":"US","id":"phi-3-small-128k-instruct","launched":"","modalities":["text"],"model":"PHI 3 Small 128k","model_family":"Phi","model_variant":"PHI 3 Small 128k Instruct","model_version":"Phi 3.5","name":"PHI 3 Small 128k Instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3-small-8k-instruct":{"country":"US","id":"phi-3-small-8k-instruct","launched":"","modalities":["text"],"model":"Phi-3 Small","model_family":"Phi","model_variant":"Phi-3 Small 8K","model_version":"Phi-3","name":"Phi-3 Small 8K Instruct","open_source":true,"parameter_size":"7B","provider":"microsoft","url":""},"phi-3-vision-128k-instruct":{"country":"US","id":"phi-3-vision-128k-instruct","launched":"2024-08-20","modalities":["visual"],"model":"Phi-3 Vision","model_family":"Phi","model_variant":"Phi-3 Vision 128K Instruct","model_version":"Phi 3","name":"Phi-3 Vision 128K Instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3-vision-4b":{"country":"US","id":"phi-3-vision-4b","launched":"2024-08-20","modalities":["visual"],"model":"Phi-3 Vision","model_family":"Phi","model_variant":"Phi-3 Vision 4B","model_version":"Phi-3","name":"Phi-3 Vision 4B","open_source":true,"parameter_size":"4B","provider":"microsoft","url":""},"phi-3.1-mini-128k-instruct":{"country":"US","id":"phi-3.1-mini-128k-instruct","launched":"","modalities":["text"],"model":"PHI 3.1 Mini 128k","model_family":"Phi","model_variant":"PHI 3.1 Mini 128k Instruct","model_version":"Phi 3.5","name":"PHI 3.1 Mini 128k Instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3.128k":{"country":"US","id":"phi-3.128k","launched":"","modalities":["text"],"model":"PHI 3.128k","model_family":"Phi","model_variant":"PHI 3.128k","model_version":"Phi 3.5","name":"PHI 3.128k","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3.5-mini-instruct":{"country":"US","id":"phi-3.5-mini-instruct","launched":"2024-08-23","modalities":["text"],"model":"Phi-3.5-mini-instruct","model_family":"Phi","model_variant":"Phi-3.5-mini","model_version":"Phi 3.5","name":"Phi-3.5-mini-instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-3.5-moe-instruct":{"country":"US","id":"phi-3.5-moe-instruct","launched":"2024-08-23","modalities":["text"],"model":"Phi 3.5 MoE","model_family":"Phi","model_variant":"Phi-3.5-MoE","model_version":"Phi 3.5 MoE","name":"Phi-3.5-MoE-instruct","open_source":true,"parameter_size":"42B","provider":"microsoft","url":""},"phi-3.5-vision-4b":{"country":"US","id":"phi-3.5-vision-4b","launched":"2024-08-20","modalities":["visual"],"model":"Phi-3.5 Vision 4B","model_family":"Phi","model_variant":"Phi-3.5 Vision 4B","model_version":"Phi 3.5","name":"Phi3.5-Vision-4B","open_source":true,"parameter_size":"4B","provider":"microsoft","url":""},"phi-3.5-vision-instruct":{"country":"US","id":"phi-3.5-vision-instruct","launched":"","modalities":["visual"],"model":"Phi-3.5 Vision","model_family":"Phi","model_variant":"Phi-3.5 Vision Instruct","model_version":"Phi 3.5","name":"Phi-3.5 Vision Instruct","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-4":{"country":"US","id":"phi-4","launched":"2025-04-30","modalities":["text"],"model":"PHI 4","model_family":"Phi","model_variant":"PHI 4","model_version":"Phi 3.5","name":"PHI 4","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-4-14b":{"country":"US","id":"phi-4-14b","launched":"2025-04-30","modalities":["text"],"model":"Phi-4 14B","model_family":"Phi","model_variant":"Phi-4-14B","model_version":"Phi 4","name":"Phi-4 14B","open_source":true,"parameter_size":"14B","provider":"microsoft","url":""},"phi-4-mini":{"country":"US","id":"phi-4-mini","launched":"2025-04-30","modalities":["text"],"model":"Phi 4 Mini","model_family":"Phi","model_variant":"Phi 4 Mini","model_version":"Phi 4","name":"Phi 4 Mini","open_source":true,"parameter_size":"","provider":"microsoft","url":""},"phi-4-mini-3.8b":{"country":"US","id":"phi-4-mini-3.8b","launched":"2025-04-30","modalities":["text"],"model":"Phi-4-mini 3.8B","model_family":"Phi","model_variant":"Phi-4-mini-3.8B","model_version":"Phi 4 mini","name":"Phi-4-mini 3.8B","open_source":true,"parameter_size":"3.8B","provider":"microsoft","url":""},"phi-4-reasoning":{"country":"US","id":"phi-4-reasoning","launched":"2025-04-30","modalities":["text"],"model":"Phi-4 Reasoning","model_family":"Phi","model_variant":"4 Reasoning","model_version":"Phi 4","name":"Phi-4 Reasoning","open_source":true,"parameter_size":"14.7B","provider":"microsoft","url":""},"phi-4-reasoning-plus":{"country":"US","id":"phi-4-reasoning-plus","launched":"2025-04-30","modalities":["text"],"model":"Phi-4 Reasoning Plus","model_family":"Phi 4","model_variant":"Phi-4 Reasoning Plus","model_version":"Phi-4","name":"Phi-4 Reasoning Plus","open_source":false,"parameter_size":"","provider":"microsoft","url":""},"phind-base":{"country":"US","id":"phind-base","launched":"","modalities":["text"],"model":"Phind Base","model_family":"Phind","model_variant":"Phind Base","model_version":"Phind","name":"Phind Base","open_source":true,"parameter_size":"34B","provider":"phind","url":""},"phind-codellama-34b-v2":{"country":"US","id":"phind-codellama-34b-v2","launched":"","modalities":["text"],"model":"Phind-CodeLlama-34B-v2","model_family":"Phind-CodeLlama-34B-v2","model_variant":"Phind-CodeLlama-34B-v2","model_version":"Phind-CodeLlama-34B-v2","name":"Phind-CodeLlama-34B-v2","open_source":true,"parameter_size":"34B","provider":"phind","url":""},"pixtral-12b":{"country":"FR","id":"pixtral-12b","launched":"2024-09-17","modalities":["visual"],"model":"Pixtral 12B","model_family":"Pixtral","model_variant":"12B","model_version":"Pixtral","name":"Pixtral-12B","open_source":true,"parameter_size":"12B","provider":"mistral","url":""},"pixtral-12b-2409":{"country":"FR","id":"pixtral-12b-2409","launched":"2024-09-01","modalities":["visual"],"model":"Pixtral 12B","model_family":"Pixtral","model_variant":"12B 2409","model_version":"Pixtral","name":"Pixtral 12B (24-09)","open_source":true,"parameter_size":"12B","provider":"mistral","url":""},"pixtral-large-2411":{"country":"FR","id":"pixtral-large-2411","launched":"2024-11-18","modalities":["visual"],"model":"Pixtral Large","model_family":"Pixtral","model_variant":"Large 2411","model_version":"Pixtral","name":"Pixtral Large (24-11)","open_source":true,"parameter_size":"","provider":"mistral","url":""},"plm-3b":{"country":"","id":"plm-3b","launched":"","modalities":["visual"],"model":"PLM 3B","model_family":"PLM","model_variant":"PLM-3B","model_version":"PLM","name":"PLM-3B","open_source":true,"parameter_size":"3B","provider":"unknown","url":""},"plm-8b":{"country":"","id":"plm-8b","launched":"","modalities":["visual"],"model":"PLM 8B","model_family":"PLM","model_variant":"PLM-8B","model_version":"PLM","name":"PLM-8B","open_source":true,"parameter_size":"8B","provider":"unknown","url":""},"points-reader":{"country":"CN","id":"points-reader","launched":"","modalities":["visual"],"model":"POINTS-Reader","model_family":"POINTS","model_variant":"Reader","model_version":"POINTS","name":"POINTS-Reader","open_source":true,"parameter_size":"","provider":"whu","url":""},"polycoder-2.7b":{"country":"US","id":"polycoder-2.7b","launched":"","modalities":["text"],"model":"PolyCoder-2.7B","model_family":"PolyCoder-2.7B","model_variant":"PolyCoder-2.7B","model_version":"PolyCoder-2.7B","name":"PolyCoder-2.7B","open_source":true,"parameter_size":"2.7B","provider":"cmu","url":""},"pp-structurev3":{"country":"CN","id":"pp-structurev3","launched":"","modalities":["visual"],"model":"PP-Structure","model_family":"PP-Structure","model_variant":"V3","model_version":"PP-Structure V3","name":"PP-StructureV3","open_source":true,"parameter_size":"","provider":"paddlepaddle","url":""},"ppstructure-v3":{"country":"CN","id":"ppstructure-v3","launched":"","modalities":["visual"],"model":"PPStructure v3","model_family":"PPStructure","model_variant":"v3","model_version":"PPStructure","name":"PPStructure v3","open_source":true,"parameter_size":"","provider":"paddlepaddle","url":""},"pythia-1.4b":{"country":"US","id":"pythia-1.4b","launched":"2023-05-03","modalities":["text"],"model":"Pythia 1.4B","model_family":"Pythia","model_variant":"1.4B","model_version":"Pythia","name":"Pythia 1.4B","open_source":"plus","parameter_size":"1.4B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-1.4b"},"pythia-12b":{"country":"US","id":"pythia-12b","launched":"2023-05-03","modalities":["text"],"model":"Pythia 12B","model_family":"Pythia","model_variant":"12B","model_version":"Pythia","name":"Pythia 12B","open_source":"plus","parameter_size":"12B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-12b"},"pythia-12b-oasst-sft":{"country":"US","id":"pythia-12b-oasst-sft","launched":"","modalities":["text"],"model":"Pythia 12B OASST SFT","model_family":"Pythia","model_variant":"12B OASST SFT","model_version":"Pythia","name":"Pythia 12B OASST SFT","open_source":true,"parameter_size":"12B","provider":"openassistant","url":""},"pythia-12b-sft":{"country":"US","id":"pythia-12b-sft","launched":"","modalities":["text"],"model":"Pythia 12B SFT","model_family":"Pythia","model_variant":"12B SFT","model_version":"Pythia","name":"Pythia 12B SFT","open_source":true,"parameter_size":"12B","provider":"eleutherai","url":""},"pythia-160m":{"country":"US","id":"pythia-160m","launched":"2023-05-03","modalities":["text"],"model":"Pythia 160M","model_family":"Pythia","model_variant":"160M","model_version":"Pythia","name":"Pythia 160M","open_source":"plus","parameter_size":"0.16B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-160m"},"pythia-1b":{"country":"US","id":"pythia-1b","launched":"2023-05-03","modalities":["text"],"model":"Pythia 1B","model_family":"Pythia","model_variant":"1B","model_version":"Pythia","name":"Pythia 1B","open_source":"plus","parameter_size":"1B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-1b"},"pythia-2.8b":{"country":"US","id":"pythia-2.8b","launched":"2023-05-03","modalities":["text"],"model":"Pythia 2.8B","model_family":"Pythia","model_variant":"2.8B","model_version":"Pythia","name":"Pythia 2.8B","open_source":"plus","parameter_size":"2.8B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-2.8b"},"pythia-410m":{"country":"US","id":"pythia-410m","launched":"2023-05-03","modalities":["text"],"model":"Pythia 410M","model_family":"Pythia","model_variant":"410M","model_version":"Pythia","name":"Pythia 410M","open_source":"plus","parameter_size":"0.41B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-410m"},"pythia-6.9b":{"country":"US","id":"pythia-6.9b","launched":"2023-05-03","modalities":["text"],"model":"Pythia 6.9B","model_family":"Pythia","model_variant":"6.9B","model_version":"Pythia","name":"Pythia 6.9B","open_source":"plus","parameter_size":"6.9B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-6.9b"},"pythia-70m":{"country":"US","id":"pythia-70m","launched":"2023-05-03","modalities":["text"],"model":"Pythia 70M","model_family":"Pythia","model_variant":"70M","model_version":"Pythia","name":"Pythia 70M","open_source":"plus","parameter_size":"0.07B","provider":"eleutherai","url":"https://huggingface.co/EleutherAI/pythia-70m"},"python-code-13b":{"country":"US","id":"python-code-13b","launched":"","modalities":["text"],"model":"Python-Code-13B","model_family":"Python-Code-13B","model_variant":"Python-Code-13B","model_version":"Python-Code-13B","name":"Python-Code-13B","open_source":true,"parameter_size":"13B","provider":"python","url":""},"quasar-alpha":{"country":"US","id":"quasar-alpha","launched":"","modalities":["text"],"model":"Quasar Alpha","model_family":"Quasar","model_variant":"Quasar Alpha","model_version":"Quasar","name":"Quasar Alpha","open_source":true,"parameter_size":"","provider":"quasar","url":""},"questa-nemotron-1.5b":{"country":"US","id":"questa-nemotron-1.5b","launched":"2025-09-28","modalities":["text"],"model":"QuestA Nemotron","model_family":"Nemotron","model_variant":"QuestA 1.5B","model_version":"QuestA Nemotron","name":"QuestA Nemotron 1.5B","open_source":true,"parameter_size":"1.5B","provider":"amazon","url":""},"qvq-72b":{"country":"CN","id":"qvq-72b","launched":"2024-12-26","modalities":["visual"],"model":"QvQ 72B","model_family":"QvQ","model_variant":"QvQ-72B","model_version":"QvQ 72B","name":"QvQ 72B","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen-1.7b-thinking":{"country":"CN","id":"qwen-1.7b-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 1.7B","model_family":"Qwen","model_variant":"Qwen-1.7B-Thinking","model_version":"Qwen3","name":"Qwen1.7B Thinking","open_source":true,"parameter_size":"1.7B","provider":"qwen","url":""},"qwen-14b":{"country":"CN","id":"qwen-14b","launched":"","modalities":["text"],"model":"Qwen 14B","model_family":"Qwen","model_variant":"Qwen 14B","model_version":"Qwen","name":"Qwen 14B","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen-14b-chat":{"country":"CN","id":"qwen-14b-chat","launched":"","modalities":["text"],"model":"Qwen 14B","model_family":"Qwen","model_variant":"Qwen 14B","model_version":"Qwen","name":"Qwen 14B Chat","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen-2-72b":{"country":"CN","id":"qwen-2-72b","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 72B","model_family":"Qwen","model_variant":"Qwen2-72B","model_version":"Qwen2","name":"Qwen2 72B","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen-2-72b-instruct":{"country":"CN","id":"qwen-2-72b-instruct","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 72B","model_family":"Qwen","model_variant":"Qwen2-72B","model_version":"Qwen2","name":"Qwen2 72B Instruct","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen-2.5-3b-instruct":{"country":"CN","id":"qwen-2.5-3b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 3B","model_family":"Qwen","model_variant":"Qwen2.5-3B Instruct","model_version":"Qwen2.5","name":"Qwen2.5 3B Instruct","open_source":true,"parameter_size":"3B","provider":"qwen","url":""},"qwen-2.5-7.6b":{"country":"CN","id":"qwen-2.5-7.6b","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 7.6B","model_family":"Qwen","model_variant":"Qwen2.5-7.6B","model_version":"Qwen2.5","name":"Qwen2.5 7.6B","open_source":true,"parameter_size":"7.6B","provider":"qwen","url":""},"qwen-2.5-72b":{"country":"CN","id":"qwen-2.5-72b","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 72B","model_family":"Qwen","model_variant":"72B","model_version":"Qwen2.5","name":"Qwen 2.5 72B","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen-2.5-max":{"country":"CN","id":"qwen-2.5-max","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 Max","model_family":"Qwen","model_variant":"Qwen2.5-Max","model_version":"Qwen2.5","name":"Qwen2.5 Max","open_source":false,"parameter_size":"100Be","provider":"qwen","url":""},"qwen-3":{"country":"CN","id":"qwen-3","launched":"2025-04-28","modalities":["text"],"model":"Qwen3","model_family":"Qwen","model_variant":"Qwen3","model_version":"Qwen3","name":"Qwen3","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen-3-0.6b":{"country":"CN","id":"qwen-3-0.6b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 0.6B","model_family":"Qwen","model_variant":"0.6B","model_version":"Qwen3","name":"Qwen 3 0.6B","open_source":true,"parameter_size":"0.6B","provider":"qwen","url":""},"qwen-3-1.7b":{"country":"CN","id":"qwen-3-1.7b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 1.7B","model_family":"Qwen","model_variant":"1.7B","model_version":"Qwen3","name":"Qwen 3 1.7B","open_source":true,"parameter_size":"1.7B","provider":"qwen","url":""},"qwen-3-235b-a22b":{"active_parameters":"22B","country":"CN","id":"qwen-3-235b-a22b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3 235B A22B","model_version":"Qwen3","name":"Qwen3 235B A22B","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen-3-235b-a22b-instruct-2507":{"active_parameters":"22B","country":"CN","id":"qwen-3-235b-a22b-instruct-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3 235B A22B","model_version":"Qwen3","name":"Qwen3 235B A22B Instruct 2507","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen-3-235b-a22b-nonthinking":{"active_parameters":"22B","country":"CN","id":"qwen-3-235b-a22b-nonthinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3-235B-A22B Non-Thinking","model_version":"Qwen3","name":"Qwen3-235B-A22B Non-Thinking","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen-3-235b-a22b-thinking":{"active_parameters":"22B","country":"CN","id":"qwen-3-235b-a22b-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3 235B A22B Thinking","model_version":"Qwen3","name":"Qwen3 235B A22B Thinking","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen-3-235b-a22b-thinking-2507":{"active_parameters":"22B","country":"CN","id":"qwen-3-235b-a22b-thinking-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3 235B A22B","model_version":"Qwen3","name":"Qwen3 235B A22B Thinking 2507","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen-3-30b-a3b":{"active_parameters":"3B","country":"CN","id":"qwen-3-30b-a3b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 30B A3B","model_family":"Qwen","model_variant":"Qwen3 30B A3B","model_version":"Qwen3","name":"Qwen3 30B A3B","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen-3-32b":{"country":"CN","id":"qwen-3-32b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 32B","model_family":"Qwen","model_variant":"Qwen3 32B","model_version":"Qwen3","name":"Qwen3 32B","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen-3-coder-480b-a35b-instruct":{"active_parameters":"35B","country":"CN","id":"qwen-3-coder-480b-a35b-instruct","launched":"2025-07-24","modalities":["text"],"model":"Qwen3 Coder 480B A35B","model_family":"Qwen","model_variant":"Qwen3 Coder 480B A35B","model_version":"Qwen3","name":"Qwen3 Coder 480B A35B Instruct","open_source":true,"parameter_size":"480B","provider":"qwen","url":""},"qwen-deepresearch-2511":{"country":"CN","id":"qwen-deepresearch-2511","launched":"","modalities":["text"],"model":"Qwen DeepResearch","model_family":"Qwen","model_variant":"DeepResearch 2511","model_version":"2511","name":"Qwen DeepResearch 2511","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen-image":{"country":"CN","id":"qwen-image","launched":"2025-08-04","modalities":["visual"],"model":"Qwen-Image","model_family":"Qwen-Image","model_variant":"Qwen-Image","model_version":"Qwen-Image","name":"Qwen-Image","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen-image-2512":{"country":"CN","id":"qwen-image-2512","launched":"2025-12-30","modalities":["visual"],"model":"Qwen-Image","model_family":"Qwen-Image","model_variant":"2512","model_version":"Qwen-Image","name":"Qwen-Image-2512","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen-max-0919":{"country":"CN","id":"qwen-max-0919","launched":"2024-09-19","modalities":["text"],"model":"Qwen Max","model_family":"Qwen","model_variant":"Qwen Max","model_version":"Qwen","name":"Qwen Max (09-19)","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen-max-2025-01-25":{"country":"CN","id":"qwen-max-2025-01-25","launched":"2025-01-25","modalities":["text"],"model":"Qwen Max","model_family":"Qwen","model_variant":"Qwen Max 2025-01-25","model_version":"Qwen Max","name":"Qwen Max (2025-01-25)","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen-plus-0125":{"country":"CN","id":"qwen-plus-0125","launched":"2025-01-25","modalities":["text"],"model":"Qwen Plus","model_family":"Qwen","model_variant":"Qwen Plus","model_version":"Qwen","name":"Qwen Plus (01-25)","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen-vl-max-1119":{"country":"CN","id":"qwen-vl-max-1119","launched":"2024-11-19","modalities":["visual"],"model":"Qwen VL Max","model_family":"Qwen","model_variant":"VL Max 1119","model_version":"Qwen VL Max","name":"Qwen VL Max (1119)","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen-vl-max-2025-08-13":{"country":"CN","id":"qwen-vl-max-2025-08-13","launched":"2025-08-13","modalities":["visual"],"model":"Qwen VL Max","model_family":"Qwen","model_variant":"VL Max 20250813","model_version":"Qwen VL Max","name":"Qwen VL Max (2025-08-13)","open_source":false,"parameter_size":"","provider":"qwen","url":"https://lmarena.ai/leaderboard/vision"},"qwen1.5-110b-chat":{"country":"CN","id":"qwen1.5-110b-chat","launched":"","modalities":["text"],"model":"Qwen1.5 110B Chat","model_family":"Qwen","model_variant":"Qwen1.5 110B Chat","model_version":"Qwen1.5","name":"Qwen1.5 110B Chat","open_source":true,"parameter_size":"110B","provider":"qwen","url":""},"qwen1.5-14b":{"country":"CN","id":"qwen1.5-14b","launched":"","modalities":["text"],"model":"Qwen1.5 14B","model_family":"Qwen","model_variant":"Qwen1.5 14B","model_version":"Qwen1.5","name":"Qwen1.5 14B","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen1.5-14b-chat":{"country":"CN","id":"qwen1.5-14b-chat","launched":"","modalities":["text"],"model":"Qwen1.5 14B","model_family":"Qwen","model_variant":"Qwen1.5 14B","model_version":"Qwen1.5","name":"Qwen1.5 14B Chat","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen1.5-32b-chat":{"country":"CN","id":"qwen1.5-32b-chat","launched":"","modalities":["text"],"model":"Qwen1.5 32B Chat","model_family":"Qwen","model_variant":"Qwen1.5 32B Chat","model_version":"Qwen1.5","name":"Qwen1.5 32B Chat","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen1.5-4b-chat":{"country":"CN","id":"qwen1.5-4b-chat","launched":"","modalities":["text"],"model":"Qwen1.5 4B","model_family":"Qwen","model_variant":"Qwen1.5 4B","model_version":"Qwen1.5","name":"Qwen1.5 4B Chat","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen1.5-72b-chat":{"country":"CN","id":"qwen1.5-72b-chat","launched":"","modalities":["text"],"model":"Qwen1.5 72B","model_family":"Qwen","model_variant":"Qwen1.5","model_version":"Qwen1.5","name":"Qwen1.5-72B-Chat","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen1.5-7b-chat":{"country":"CN","id":"qwen1.5-7b-chat","launched":"","modalities":["text"],"model":"Qwen1.5 7B","model_family":"Qwen","model_variant":"Qwen1.5 7B","model_version":"Qwen1.5","name":"Qwen1.5 7B Chat","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2-72b":{"country":"CN","id":"qwen2-72b","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 72B","model_family":"Qwen","model_variant":"Qwen2 72B","model_version":"Qwen2","name":"Qwen2 72B","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2-72b-chat":{"country":"CN","id":"qwen2-72b-chat","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 72B Chat","model_family":"Qwen","model_variant":"Qwen2 72B Chat","model_version":"Qwen2","name":"Qwen2 72B Chat","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2-72b-instruct":{"country":"CN","id":"qwen2-72b-instruct","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 72B","model_family":"Qwen","model_variant":"Qwen2 72B","model_version":"Qwen2","name":"Qwen2 72B Instruct","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2-7b":{"country":"CN","id":"qwen2-7b","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 7B","model_family":"Qwen","model_variant":"Qwen2 7B","model_version":"Qwen2","name":"Qwen2 7B","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2-7b-instruct":{"country":"CN","id":"qwen2-7b-instruct","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 7B","model_family":"Qwen","model_variant":"Qwen2 7B Instruct","model_version":"Qwen2","name":"Qwen2 7B Instruct","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2-vl":{"country":"CN","id":"qwen2-vl","launched":"","modalities":["visual"],"model":"Qwen2 VL","model_family":"Qwen","model_variant":"Qwen2 VL","model_version":"Qwen2 VL","name":"Qwen2 VL","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen2-vl-2b":{"country":"CN","id":"qwen2-vl-2b","launched":"","modalities":["visual"],"model":"Qwen2-VL 2B","model_family":"Qwen","model_variant":"Qwen2-VL-2B","model_version":"Qwen2 VL","name":"Qwen2-VL-2B","open_source":true,"parameter_size":"2B","provider":"qwen","url":""},"qwen2-vl-72b":{"country":"CN","id":"qwen2-vl-72b","launched":"","modalities":["visual"],"model":"Qwen2-VL 72B","model_family":"Qwen","model_variant":"Qwen2-VL-72B","model_version":"Qwen2 VL","name":"Qwen2-VL-72B","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2-vl-72b-instruct":{"country":"CN","id":"qwen2-vl-72b-instruct","launched":"","modalities":["visual"],"model":"Qwen2-VL 72B","model_family":"Qwen","model_variant":"Qwen2-VL-72B-Instruct","model_version":"Qwen2 VL","name":"Qwen2-VL 72B Instruct","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2-vl-7b":{"country":"CN","id":"qwen2-vl-7b","launched":"","modalities":["visual"],"model":"Qwen2-VL 7B","model_family":"Qwen","model_variant":"Qwen2-VL-7B","model_version":"Qwen2 VL","name":"Qwen2-VL-7B","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2-vl-7b-instruct":{"country":"CN","id":"qwen2-vl-7b-instruct","launched":"","modalities":["visual"],"model":"Qwen2-VL 7B","model_family":"Qwen","model_variant":"Qwen2-VL-7B-Instruct","model_version":"Qwen2 VL","name":"Qwen2-VL 7B Instruct","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2.5-0.5b":{"country":"CN","id":"qwen2.5-0.5b","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 0.5B","model_family":"Qwen","model_variant":"Qwen2.5-0.5B","model_version":"Qwen2.5","name":"Qwen2.5-0.5B","open_source":true,"parameter_size":"0.5B","provider":"qwen","url":""},"qwen2.5-0.5b-instruct":{"country":"CN","id":"qwen2.5-0.5b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 0.5B","model_family":"Qwen","model_variant":"Qwen2.5 0.5B Instruct","model_version":"Qwen2.5","name":"Qwen2.5 0.5B Instruct","open_source":true,"parameter_size":"0.5B","provider":"qwen","url":""},"qwen2.5-1-5b-instruct":{"country":"CN","id":"qwen2.5-1-5b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 1.5B Instruct","model_family":"Qwen","model_variant":"Qwen2.5 1.5B Instruct","model_version":"Qwen2.5","name":"Qwen2.5 1.5B Instruct","open_source":true,"parameter_size":"1.5B","provider":"qwen","url":""},"qwen2.5-1.5b":{"country":"CN","id":"qwen2.5-1.5b","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 1.5B","model_family":"Qwen","model_variant":"Qwen2.5-1.5B","model_version":"Qwen2.5","name":"Qwen2.5-1.5B","open_source":true,"parameter_size":"1.5B","provider":"qwen","url":""},"qwen2.5-1.5b-instruct":{"country":"CN","id":"qwen2.5-1.5b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 1.5B","model_family":"Qwen","model_variant":"Qwen2.5-1.5B-Instruct","model_version":"Qwen2.5","name":"Qwen2.5 1.5B Instruct","open_source":true,"parameter_size":"1.5B","provider":"qwen","url":""},"qwen2.5-1.5b-pretrained":{"country":"CN","id":"qwen2.5-1.5b-pretrained","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 1.5B","model_family":"Qwen","model_variant":"Qwen2.5-1.5B-Pretrained","model_version":"Qwen2.5","name":"Qwen2.5 1.5B Pretrained","open_source":true,"parameter_size":"1.5B","provider":"qwen","url":""},"qwen2.5-14b-instruct":{"country":"CN","id":"qwen2.5-14b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 14B","model_family":"Qwen","model_variant":"Qwen2.5-14B","model_version":"Qwen2.5","name":"Qwen2.5 14B Instruct","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen2.5-32b-base":{"country":"CN","id":"qwen2.5-32b-base","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 32B Base","model_family":"Qwen","model_variant":"Qwen2.5-32B Base","model_version":"Qwen2.5","name":"Qwen2.5 32B Base","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen2.5-32b-instruct":{"country":"CN","id":"qwen2.5-32b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 32B","model_family":"Qwen","model_variant":"Qwen2.5-32B","model_version":"Qwen2.5","name":"Qwen2.5 32B Instruct","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen2.5-3b":{"country":"CN","id":"qwen2.5-3b","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 3B","model_family":"Qwen","model_variant":"Qwen2.5-3B","model_version":"Qwen2.5","name":"Qwen2.5-3B","open_source":true,"parameter_size":"3B","provider":"qwen","url":""},"qwen2.5-72b-instruct":{"country":"CN","id":"qwen2.5-72b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 72B","model_family":"Qwen","model_variant":"Qwen2.5-72B","model_version":"Qwen2.5","name":"Qwen2.5-72B-Instruct","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2.5-72b-instruct-turbo":{"country":"CN","id":"qwen2.5-72b-instruct-turbo","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 72B","model_family":"Qwen","model_variant":"Qwen2.5 72B Instruct Turbo","model_version":"Qwen2.5","name":"Qwen2.5 72B Instruct Turbo","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen2.5-7b":{"country":"CN","id":"qwen2.5-7b","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 7B","model_family":"Qwen","model_variant":"Qwen2.5-7B","model_version":"Qwen2.5","name":"Qwen2.5-7B","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2.5-7b-instruct":{"country":"CN","id":"qwen2.5-7b-instruct","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 7B Instruct","model_family":"Qwen","model_variant":"Qwen2.5 7B Instruct","model_version":"Qwen2.5","name":"Qwen2.5 7B Instruct","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2.5-coder-1.5b-instruct":{"country":"CN","id":"qwen2.5-coder-1.5b-instruct","launched":"2024-11-04","modalities":["text"],"model":"Qwen2.5 Coder 1.5B Instruct","model_family":"Qwen","model_variant":"Qwen2.5 Coder 1.5B Instruct","model_version":"Qwen2.5","name":"Qwen2.5 Coder 1.5B Instruct","open_source":true,"parameter_size":"1.5B","provider":"qwen","url":""},"qwen2.5-coder-14b-instruct":{"country":"CN","id":"qwen2.5-coder-14b-instruct","launched":"2024-11-04","modalities":["text"],"model":"Qwen2.5 Coder 14B","model_family":"Qwen","model_variant":"Qwen2.5 Coder 14B Instruct","model_version":"Qwen2.5","name":"Qwen2.5 Coder 14B Instruct","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen2.5-coder-32b-instruct":{"country":"CN","id":"qwen2.5-coder-32b-instruct","launched":"2024-11-04","modalities":["text"],"model":"Qwen2.5 Coder 32B","model_family":"Qwen","model_variant":"Qwen2.5-Coder-32B","model_version":"Qwen2.5","name":"Qwen2.5-Coder 32B Instruct","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen2.5-coder-32b-instruct-hyperbolic":{"country":"CN","id":"qwen2.5-coder-32b-instruct-hyperbolic","launched":"2024-11-04","modalities":["text"],"model":"Qwen2.5 Coder 32B","model_family":"Qwen","model_variant":"Qwen2.5-Coder-32B Hyperbolic","model_version":"Qwen2.5","name":"Qwen2.5-Coder 32B Instruct (Hyperbolic)","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen2.5-coder-7b-instruct":{"country":"CN","id":"qwen2.5-coder-7b-instruct","launched":"2024-11-04","modalities":["text"],"model":"Qwen2.5-Coder 7B","model_family":"Qwen","model_variant":"Qwen2.5-Coder-7B","model_version":"Qwen2.5","name":"Qwen2.5-Coder 7B Instruct","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2.5-max":{"country":"CN","id":"qwen2.5-max","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 Max","model_family":"Qwen","model_variant":"Qwen2.5 Max","model_version":"Qwen2.5","name":"Qwen2.5 Max","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen2.5-omni":{"country":"CN","id":"qwen2.5-omni","launched":"2025-03-27","modalities":["text","visual"],"model":"Qwen2.5 Omni","model_family":"Qwen","model_variant":"Qwen2.5-Omni","model_version":"Qwen2.5 Omni","name":"Qwen2.5-Omni","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen2.5-omni-3b":{"country":"CN","id":"qwen2.5-omni-3b","launched":"2025-03-27","modalities":["text","audio","visual"],"model":"Qwen2.5-Omni-3B","model_family":"Qwen","model_variant":"Qwen2.5-Omni-3B","model_version":"Qwen2.5 Omni","name":"Qwen2.5-Omni-3B","open_source":true,"parameter_size":"5B","provider":"qwen","url":""},"qwen2.5-omni-7b":{"country":"CN","id":"qwen2.5-omni-7b","launched":"2025-03-27","modalities":["visual","text"],"model":"Qwen2.5-Omni-7B","model_family":"Qwen","model_variant":"Qwen2.5-Omni-7B","model_version":"Qwen2.5 Omni","name":"Qwen2.5-Omni-7B","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2.5-plus-1127":{"country":"CN","id":"qwen2.5-plus-1127","launched":"2024-09-19","modalities":["text"],"model":"Qwen2.5 Plus","model_family":"Qwen","model_variant":"Qwen2.5 Plus","model_version":"Qwen2.5","name":"Qwen2.5 Plus (11-27)","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen2.5-vl-32b-instruct":{"country":"CN","id":"qwen2.5-vl-32b-instruct","launched":"2025-01-27","modalities":["visual"],"model":"Qwen2.5-VL 32B","model_family":"Qwen","model_variant":"Qwen2.5-VL-32B","model_version":"Qwen2.5 VL","name":"Qwen2.5-VL 32B Instruct","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen2.5-vl-3b":{"country":"CN","id":"qwen2.5-vl-3b","launched":"2025-01-27","modalities":["visual"],"model":"Qwen2.5-VL 3B","model_family":"Qwen","model_variant":"Qwen2.5-3B","model_version":"Qwen2.5 VL","name":"Qwen2.5-VL 3B","open_source":true,"parameter_size":"3B","provider":"qwen","url":""},"qwen2.5-vl-72b":{"country":"CN","id":"qwen2.5-vl-72b","launched":"2025-01-27","modalities":["visual"],"model":"Qwen2.5-VL 72B","model_family":"Qwen","model_variant":"Qwen2.5-72B","model_version":"Qwen2.5 VL","name":"Qwen2.5-VL 72B","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2.5-vl-72b-instruct":{"country":"CN","id":"qwen2.5-vl-72b-instruct","launched":"2025-01-27","modalities":["visual"],"model":"Qwen2.5-VL 72B","model_family":"Qwen","model_variant":"Qwen2.5-VL-72B-Instruct","model_version":"Qwen2.5 VL","name":"Qwen2.5-VL 72B Instruct","open_source":true,"parameter_size":"72B","provider":"qwen","url":""},"qwen2.5-vl-7b":{"country":"CN","id":"qwen2.5-vl-7b","launched":"2025-01-27","modalities":["visual"],"model":"Qwen2.5-VL 7B","model_family":"Qwen","model_variant":"Qwen2.5-7B","model_version":"Qwen2.5 VL","name":"Qwen2.5-VL 7B","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2.5-vl-7b-instruct-rl":{"country":"CN","id":"qwen2.5-vl-7b-instruct-rl","launched":"2025-01-27","modalities":["visual"],"model":"Qwen2.5 VL 7B","model_family":"Qwen","model_variant":"Qwen2.5-VL-7B-Instruct-RL","model_version":"Qwen2.5 VL","name":"Qwen2.5-VL-7B-Instruct-RL","open_source":true,"parameter_size":"7B","provider":"qwen","url":""},"qwen2.57b-a14b":{"country":"CN","id":"qwen2.57b-a14b","launched":"2024-06-01","modalities":["text"],"model":"Qwen2 57B A14B","model_family":"Qwen","model_variant":"Qwen2 57B A14B","model_version":"Qwen2","name":"Qwen 2.57B A 14B","open_source":true,"parameter_size":"57B","provider":"qwen","url":""},"qwen2.5vl-3b":{"country":"CN","id":"qwen2.5vl-3b","launched":"2025-01-27","modalities":["visual"],"model":"Qwen2.5-VL 3B","model_family":"Qwen","model_variant":"Qwen2.5VL 3B","model_version":"Qwen2.5 VL","name":"Qwen2.5VL 3B","open_source":true,"parameter_size":"3B","provider":"qwen","url":""},"qwen3-0.6b":{"country":"CN","id":"qwen3-0.6b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 0.6B","model_family":"Qwen","model_variant":"Qwen3-0.6B","model_version":"Qwen3","name":"Qwen3-0.6B","open_source":true,"parameter_size":"0.6B","provider":"qwen","url":""},"qwen3-0.6b-base":{"country":"CN","id":"qwen3-0.6b-base","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 0.6B","model_family":"Qwen","model_variant":"Qwen3-0.6B-Base","model_version":"Qwen3","name":"Qwen3-0.6B-Base","open_source":true,"parameter_size":"0.6B","provider":"qwen","url":""},"qwen3-0.8b":{"country":"CN","id":"qwen3-0.8b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 0.8B","model_family":"Qwen","model_variant":"Qwen3-0.8B","model_version":"Qwen3","name":"Qwen3-0.8B","open_source":true,"parameter_size":"0.8B","provider":"qwen","url":""},"qwen3-1.7b":{"country":"CN","id":"qwen3-1.7b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 1.7B","model_family":"Qwen","model_variant":"Qwen3-1.7B","model_version":"Qwen3","name":"Qwen3-1.7B","open_source":true,"parameter_size":"1.7B","provider":"qwen","url":""},"qwen3-1.7b-thinking":{"country":"CN","id":"qwen3-1.7b-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 1.7B","model_family":"Qwen","model_variant":"Qwen3-1.7B Thinking","model_version":"Qwen3","name":"Qwen3-1.7B Thinking","open_source":true,"parameter_size":"1.7B","provider":"qwen","url":""},"qwen3-14b":{"country":"CN","id":"qwen3-14b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 14B","model_family":"Qwen","model_variant":"Qwen3-14B","model_version":"Qwen3","name":"Qwen3-14B","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen3-14b-base":{"country":"CN","id":"qwen3-14b-base","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 14B","model_family":"Qwen","model_variant":"Qwen3 14B Base","model_version":"Qwen3","name":"Qwen3 14B Base","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen3-14b-thinking":{"country":"CN","id":"qwen3-14b-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 14B","model_family":"Qwen","model_variant":"Qwen3-14B Thinking","model_version":"Qwen3","name":"Qwen3-14B Thinking","open_source":true,"parameter_size":"14B","provider":"qwen","url":""},"qwen3-235b-2507":{"active_parameters":"22B","country":"CN","id":"qwen3-235b-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3 235B 2507","model_version":"Qwen3","name":"Qwen3 235B (2507)","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-235b-a22b":{"active_parameters":"22B","country":"CN","id":"qwen3-235b-a22b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3-235B-A22B","model_version":"Qwen3","name":"Qwen3-235B-A22B","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-235b-a22b-2507":{"active_parameters":"22B","country":"CN","id":"qwen3-235b-a22b-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3-235B-A22B","model_version":"Qwen3","name":"Qwen3-235B-A22B (2507)","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-235b-a22b-base":{"active_parameters":"22B","country":"CN","id":"qwen3-235b-a22b-base","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3-235B-A22B-Base","model_version":"Qwen3","name":"Qwen3-235B-A22B Base","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-235b-a22b-instruct-2507":{"active_parameters":"22B","country":"CN","id":"qwen3-235b-a22b-instruct-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3-235B-A22B","model_version":"Qwen3","name":"Qwen3-235B-A22B-Instruct-2507","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-235b-a22b-no-thinking":{"active_parameters":"22B","country":"CN","id":"qwen3-235b-a22b-no-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3 235B A22B Non-Thinking","model_version":"Qwen3","name":"Qwen3 235B A22B No-Thinking","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-235b-a22b-thinking-2507":{"active_parameters":"22B","country":"CN","id":"qwen3-235b-a22b-thinking-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 235B A22B","model_family":"Qwen","model_variant":"Qwen3-235B-A22B","model_version":"Qwen3","name":"Qwen3-235B-A22B-Thinking-2507","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-256b":{"country":"CN","id":"qwen3-256b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 256B","model_family":"Qwen","model_variant":"256B","model_version":"Qwen3","name":"Qwen3 256B","open_source":false,"parameter_size":"256B","provider":"qwen","url":""},"qwen3-30b-a3b":{"active_parameters":"3B","country":"CN","id":"qwen3-30b-a3b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 30B A3B","model_family":"Qwen","model_variant":"Qwen3-30B-A3B","model_version":"Qwen3","name":"Qwen3-30B-A3B","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-30b-a3b-2507":{"active_parameters":"3B","country":"CN","id":"qwen3-30b-a3b-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 30B A3B","model_family":"Qwen","model_variant":"Qwen3-30B-A3B-2507","model_version":"Qwen3","name":"Qwen3-30B-A3B-2507","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-30b-a3b-base":{"active_parameters":"3B","country":"CN","id":"qwen3-30b-a3b-base","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 30B A3B","model_family":"Qwen","model_variant":"Qwen3-30B-A3B-Base","model_version":"Qwen3","name":"Qwen3-30B-A3B-Base","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-30b-a3b-instruct-2507":{"active_parameters":"3B","country":"CN","id":"qwen3-30b-a3b-instruct-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 30B A3B","model_family":"Qwen","model_variant":"Qwen3-30B-A3B-Instruct-2507","model_version":"Qwen3","name":"Qwen3-30B-A3B-Instruct-2507","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-30b-a3b-thinking-2507":{"country":"CN","id":"qwen3-30b-a3b-thinking-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 30B A3B","model_family":"Qwen","model_variant":"Qwen3-30B-A3B-Thinking-2507","model_version":"Qwen3","name":"Qwen3-30B-A3B-Thinking-2507","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-32b":{"country":"CN","id":"qwen3-32b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 32B","model_family":"Qwen","model_variant":"Qwen3-32B","model_version":"Qwen3","name":"Qwen3-32B","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen3-32b-nothinking":{"country":"CN","id":"qwen3-32b-nothinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 32B","model_family":"Qwen","model_variant":"Qwen3-32B Non-Thinking","model_version":"Qwen3","name":"Qwen3-32B Non-Thinking","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen3-32b-thinking":{"country":"CN","id":"qwen3-32b-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 32B","model_family":"Qwen","model_variant":"Qwen3-32B Thinking","model_version":"Qwen3","name":"Qwen3-32B Thinking","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen3-32b-thinking-2504":{"country":"CN","id":"qwen3-32b-thinking-2504","launched":"","modalities":["text"],"model":"Qwen3 32B","model_family":"Qwen","model_variant":"32B Thinking 2025-04","model_version":"Qwen3","name":"Qwen3 32B Thinking (2025-04)","open_source":false,"parameter_size":"32B","provider":"qwen","url":""},"qwen3-4b":{"country":"CN","id":"qwen3-4b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B","model_family":"Qwen","model_variant":"Qwen3-4B","model_version":"Qwen3","name":"Qwen3-4B","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen3-4b-2507":{"country":"CN","id":"qwen3-4b-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B","model_family":"Qwen","model_variant":"Qwen3 4B 2507","model_version":"Qwen3","name":"Qwen3 4B (2507)","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen3-4b-base":{"country":"CN","id":"qwen3-4b-base","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B","model_family":"Qwen","model_variant":"Qwen3-4B","model_version":"Qwen3","name":"Qwen3-4B-Base","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen3-4b-instruct-2507":{"country":"CN","id":"qwen3-4b-instruct-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B","model_family":"Qwen","model_variant":"Qwen3-4B-Instruct-2507","model_version":"Qwen3","name":"Qwen3-4B-Instruct-2507","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen3-4b-saferl":{"country":"CN","id":"qwen3-4b-saferl","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B SafeRL","model_family":"Qwen","model_variant":"Qwen3-4B-SafeRL","model_version":"Qwen3","name":"Qwen3-4B-SafeRL","open_source":true,"parameter_size":"4.41B","provider":"qwen","url":""},"qwen3-4b-saferl-thinking":{"country":"CN","id":"qwen3-4b-saferl-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B SafeRL","model_family":"Qwen","model_variant":"Qwen3-4B-SafeRL Thinking","model_version":"Qwen3","name":"Qwen3-4B-SafeRL Thinking","open_source":true,"parameter_size":"4.41B","provider":"qwen","url":""},"qwen3-4b-thinking":{"country":"CN","id":"qwen3-4b-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B","model_family":"Qwen","model_variant":"Qwen3-4B Thinking","model_version":"Qwen3","name":"Qwen3-4B Thinking","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen3-4b-thinking-2507":{"country":"CN","id":"qwen3-4b-thinking-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 4B","model_family":"Qwen","model_variant":"Qwen3-4B-Thinking-2507","model_version":"Qwen3","name":"Qwen3-4B Thinking 2507","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen3-6b":{"country":"CN","id":"qwen3-6b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 6B","model_family":"Qwen","model_variant":"Qwen3-6B","model_version":"Qwen3","name":"Qwen3-6B","open_source":true,"parameter_size":"6B","provider":"qwen","url":""},"qwen3-8b":{"country":"CN","id":"qwen3-8b","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 8B","model_family":"Qwen","model_variant":"Qwen3-8B","model_version":"Qwen3","name":"Qwen3-8B","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3-8b-base":{"country":"CN","id":"qwen3-8b-base","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 8B","model_family":"Qwen","model_variant":"Qwen3 8B Base","model_version":"Qwen3","name":"Qwen3 8B Base","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3-8b-instruct":{"country":"CN","id":"qwen3-8b-instruct","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 8B","model_family":"Qwen","model_variant":"Qwen3-8B-Instruct","model_version":"Qwen3","name":"Qwen3-8B-Instruct","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3-8b-non-thinking":{"country":"CN","id":"qwen3-8b-non-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 8B","model_family":"Qwen","model_variant":"Qwen3-8B Non-Thinking","model_version":"Qwen3","name":"Qwen3-8B Non-Thinking","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3-8b-nothinking-2504":{"country":"CN","id":"qwen3-8b-nothinking-2504","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 8B","model_family":"Qwen","model_variant":"Qwen3-8B-NonThinking-2504","model_version":"Qwen3","name":"Qwen3-8B-NonThinking-2504","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3-8b-thinking":{"country":"CN","id":"qwen3-8b-thinking","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 8B","model_family":"Qwen","model_variant":"Qwen3-8B Thinking","model_version":"Qwen3","name":"Qwen3-8B Thinking","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3-coder":{"country":"CN","id":"qwen3-coder","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Coder","model_family":"Qwen","model_variant":"Qwen3 Coder","model_version":"Qwen3","name":"Qwen3 Coder","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen3-coder-30b-a3b-instruct":{"active_parameters":"3B","country":"CN","id":"qwen3-coder-30b-a3b-instruct","launched":"2025-07-24","modalities":["text"],"model":"Qwen3 Coder 30B A3B","model_family":"Qwen","model_variant":"Qwen3-Coder-30B-A3B Instruct","model_version":"Qwen3","name":"Qwen3-Coder-30B-A3B Instruct","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-coder-480b":{"active_parameters":"35B","country":"CN","id":"qwen3-coder-480b","launched":"2025-07-24","modalities":["text"],"model":"Qwen3 Coder 480B","model_family":"Qwen","model_variant":"Qwen3-Coder-480B","model_version":"Qwen3","name":"Qwen3-Coder-480B","open_source":true,"parameter_size":"480B","provider":"qwen","url":""},"qwen3-coder-480b-a35b":{"country":"CN","id":"qwen3-coder-480b-a35b","launched":"2025-07-24","modalities":["text"],"model":"Qwen3 Coder 480B A35B","model_family":"Qwen","model_variant":"Qwen3-480B","model_version":"Qwen3","name":"Qwen3-Coder-480B-A35B","open_source":true,"parameter_size":"480B","provider":"qwen","url":""},"qwen3-coder-480b-a35b-instruct":{"country":"CN","id":"qwen3-coder-480b-a35b-instruct","launched":"2025-07-24","modalities":["text"],"model":"Qwen3 Coder 480B A35B","model_family":"Qwen","model_variant":"Qwen3-480B Instruct","model_version":"Qwen3","name":"Qwen3-Coder-480B-A35B-Instruct","open_source":true,"parameter_size":"480B","provider":"qwen","url":""},"qwen3-coder-480b-a35b-mini-swe-agent":{"country":"CN","id":"qwen3-coder-480b-a35b-mini-swe-agent","launched":"2025-07-24","modalities":["text"],"model":"Qwen3 Coder 480B A35B","model_family":"Qwen","model_variant":"Qwen3-480B mini-SWE-Agent","model_version":"Qwen3","name":"Qwen3-Coder-480B-A35B (mini-SWE-Agent)","open_source":true,"parameter_size":"480B","provider":"qwen","url":""},"qwen3-coder-next":{"active_parameters":"3B","country":"CN","id":"qwen3-coder-next","launched":"2026-02-03","modalities":["text"],"model":"Qwen3 Coder Next","model_family":"Qwen","model_variant":"Qwen3-Coder-Next","model_version":"Qwen3","name":"Qwen3-Coder-Next","open_source":true,"parameter_size":"80B","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3-Coder-Next"},"qwen3-coder-plus":{"country":"CN","id":"qwen3-coder-plus","launched":"2025-07-24","modalities":["text"],"model":"Qwen3 Coder Plus","model_family":"Qwen","model_variant":"Coder Plus","model_version":"Qwen3","name":"Qwen3 Coder Plus","open_source":true,"parameter_size":"480B","provider":"qwen","url":""},"qwen3-coder-plus-2025-09-23":{"country":"CN","id":"qwen3-coder-plus-2025-09-23","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Coder Plus","model_family":"Qwen","model_variant":"Qwen3 Coder Plus","model_version":"Qwen3","name":"Qwen3 Coder Plus (2025-09-23)","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen3-max":{"country":"CN","id":"qwen3-max","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Max","model_family":"Qwen","model_variant":"Qwen3 Max","model_version":"Qwen3","name":"Qwen3 Max","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen3-max-2025-09-23":{"country":"CN","id":"qwen3-max-2025-09-23","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Max","model_family":"Qwen","model_variant":"Qwen3 Max","model_version":"Qwen3","name":"Qwen3 Max (2025-09-23)","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen3-max-instruct-preview":{"country":"CN","id":"qwen3-max-instruct-preview","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Max","model_family":"Qwen","model_variant":"Qwen3","model_version":"Qwen3","name":"Qwen3-Max-Instruct-Preview","open_source":false,"parameter_size":"1000Be","provider":"qwen","url":""},"qwen3-max-preview":{"country":"CN","id":"qwen3-max-preview","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Max","model_family":"Qwen","model_variant":"Qwen3 Max Preview","model_version":"Qwen3","name":"Qwen3 Max Preview","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen3-max-thinking":{"country":"CN","id":"qwen3-max-thinking","launched":"2026-01-26","modalities":["text"],"model":"Qwen3 Max","model_family":"Qwen","model_variant":"Qwen3 Max Thinking","model_version":"Qwen3","name":"Qwen3 Max Thinking","open_source":false,"parameter_size":"1000B","provider":"qwen","url":""},"qwen3-max-thinking-preview":{"country":"CN","id":"qwen3-max-thinking-preview","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Max","model_family":"Qwen","model_variant":"Qwen3 Max Thinking Preview","model_version":"Qwen3","name":"Qwen3-Max-Thinking-Preview","open_source":false,"parameter_size":"","provider":"qwen","url":""},"qwen3-moe-2507":{"country":"CN","id":"qwen3-moe-2507","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 MoE","model_family":"Qwen","model_variant":"Qwen3","model_version":"Qwen3","name":"Qwen3 MoE-2507","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-next":{"country":"CN","id":"qwen3-next","launched":"2025-04-28","modalities":["text"],"model":"Qwen3 Next","model_family":"Qwen","model_variant":"Qwen3 Next","model_version":"Qwen3","name":"Qwen3 Next","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen3-next-80b-a3b":{"active_parameters":"3B","country":"CN","id":"qwen3-next-80b-a3b","launched":"2025-09-09","modalities":["text"],"model":"Qwen3 Next 80B","model_family":"Qwen","model_variant":"Qwen3 Next 80B A3B","model_version":"Qwen3 Next","name":"Qwen3 Next 80B A3B","open_source":true,"parameter_size":"80B","provider":"qwen","url":""},"qwen3-next-80b-a3b-instruct":{"active_parameters":"3B","country":"CN","id":"qwen3-next-80b-a3b-instruct","launched":"2025-09-09","modalities":["text"],"model":"Qwen3 Next 80B A3B","model_family":"Qwen","model_variant":"Qwen3-Next-80B-A3B-Instruct","model_version":"Qwen3 Next","name":"Qwen3-Next-80B-A3B-Instruct","open_source":true,"parameter_size":"80B","provider":"qwen","url":""},"qwen3-next-80b-a3b-thinking":{"active_parameters":"3B","country":"CN","id":"qwen3-next-80b-a3b-thinking","launched":"2025-09-09","modalities":["text"],"model":"Qwen3 Next 80B A3B","model_family":"Qwen","model_variant":"Qwen3-Next-80B-A3B-Thinking","model_version":"Qwen3 Next","name":"Qwen3-Next-80B-A3B-Thinking","open_source":true,"parameter_size":"80B","provider":"qwen","url":""},"qwen3-omni-30b-a3b":{"active_parameters":"3B","country":"CN","id":"qwen3-omni-30b-a3b","launched":"2025-04-28","modalities":["visual","text","speech"],"model":"Qwen3 Omni 30B A3B","model_family":"Qwen","model_variant":"Qwen3-Omni-30B-A3B","model_version":"Qwen3","name":"Qwen3-Omni-30B-A3B","open_source":true,"parameter_size":"30B","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct"},"qwen3-omni-30b-a3b-instruct":{"active_parameters":"3B","country":"CN","id":"qwen3-omni-30b-a3b-instruct","launched":"2025-04-28","modalities":["visual","text","speech"],"model":"Qwen3 Omni 30B A3B","model_family":"Qwen","model_variant":"Qwen3-Omni-30B-A3B-Instruct","model_version":"Qwen3","name":"Qwen3-Omni-30B-A3B-Instruct","open_source":true,"parameter_size":"30B","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct"},"qwen3-omni-30b-a3b-thinking":{"active_parameters":"3B","country":"CN","id":"qwen3-omni-30b-a3b-thinking","launched":"2025-04-28","modalities":["visual","text","speech"],"model":"Qwen3 Omni 30B A3B","model_family":"Qwen","model_variant":"Qwen3-Omni-30B-A3B-Thinking","model_version":"Qwen3","name":"Qwen3-Omni-30B-A3B-Thinking","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-omni-flash-instruct":{"active_parameters":"","country":"CN","id":"qwen3-omni-flash-instruct","launched":"2025-04-28","modalities":["visual","text","speech"],"model":"Qwen3 Omni Flash","model_family":"Qwen","model_variant":"Qwen3-Omni-Flash-Instruct","model_version":"Qwen3","name":"Qwen3-Omni-Flash-Instruct","open_source":true,"parameter_size":"","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3-Omni-Flash-Instruct"},"qwen3-omni-flash-thinking":{"active_parameters":"","country":"CN","id":"qwen3-omni-flash-thinking","launched":"2025-04-28","modalities":["visual","text","speech"],"model":"Qwen3 Omni Flash","model_family":"Qwen","model_variant":"Qwen3-Omni-Flash-Thinking","model_version":"Qwen3","name":"Qwen3-Omni-Flash-Thinking","open_source":true,"parameter_size":"","provider":"qwen","url":""},"qwen3-vl-235b-a22b":{"active_parameters":"22B","country":"CN","id":"qwen3-vl-235b-a22b","launched":"2025-09-24","modalities":["visual"],"model":"Qwen3-VL 235B A22B","model_family":"Qwen","model_variant":"Qwen3-VL-235B-A22B","model_version":"Qwen3 VL","name":"Qwen3-VL-235B-A22B","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-vl-235b-a22b-instruct":{"active_parameters":"22B","country":"CN","id":"qwen3-vl-235b-a22b-instruct","launched":"2025-09-24","modalities":["visual"],"model":"Qwen3-VL 235B A22B","model_family":"Qwen","model_variant":"Qwen3-VL-235B-A22B-Instruct","model_version":"Qwen3 VL","name":"Qwen3-VL-235B-A22B Instruct","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-vl-235b-a22b-thinking":{"active_parameters":"22B","country":"CN","id":"qwen3-vl-235b-a22b-thinking","launched":"2025-09-24","modalities":["visual"],"model":"Qwen3-VL 235B A22B","model_family":"Qwen","model_variant":"Qwen3-VL-235B-A22B-Thinking","model_version":"Qwen3 VL","name":"Qwen3-VL-235B-A22B Thinking","open_source":true,"parameter_size":"235B","provider":"qwen","url":""},"qwen3-vl-2b-instruct":{"country":"CN","id":"qwen3-vl-2b-instruct","launched":"2025-10-21","modalities":["visual"],"model":"Qwen3-VL 2B","model_family":"Qwen","model_variant":"Qwen3-VL 2B Instruct","model_version":"Qwen3 VL","name":"Qwen3-VL 2B Instruct","open_source":true,"parameter_size":"2B","provider":"qwen","url":""},"qwen3-vl-30b-a3b-instruct":{"active_parameters":"3B","country":"CN","id":"qwen3-vl-30b-a3b-instruct","launched":"2025-09-30","modalities":["visual"],"model":"Qwen3 VL 30B A4B","model_family":"Qwen","model_variant":"Qwen3-VL-30B-A3B-Instruct","model_version":"Qwen3 VL","name":"Qwen3-VL-30B-A3B Instruct","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-vl-30b-a3b-thinking":{"active_parameters":"3B","country":"CN","id":"qwen3-vl-30b-a3b-thinking","launched":"2025-09-30","modalities":["visual"],"model":"Qwen3 VL 30B","model_family":"Qwen","model_variant":"Qwen3-VL-30B-A3B Thinking","model_version":"Qwen3 VL","name":"Qwen3-VL-30B-A3B Thinking","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwen3-vl-32b-instruct":{"country":"CN","id":"qwen3-vl-32b-instruct","launched":"2025-10-23","modalities":["visual"],"model":"Qwen3-VL 32B","model_family":"Qwen","model_variant":"Qwen3-VL 32B Instruct","model_version":"Qwen3 VL","name":"Qwen3-VL 32B Instruct","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen3-vl-32b-thinking":{"country":"CN","id":"qwen3-vl-32b-thinking","launched":"2025-10-23","modalities":["visual"],"model":"Qwen3-VL 32B Thinking","model_family":"Qwen","model_variant":"Qwen3-VL 32B Thinking","model_version":"Qwen3 VL","name":"Qwen3-VL 32B Thinking","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwen3-vl-4b-instruct":{"country":"CN","id":"qwen3-vl-4b-instruct","launched":"2025-10-14","modalities":["visual"],"model":"Qwen3-VL 4B","model_family":"Qwen","model_variant":"Qwen3-VL-4B-Instruct","model_version":"Qwen3 VL","name":"Qwen3-VL-4B-Instruct","open_source":true,"parameter_size":"4B","provider":"qwen","url":""},"qwen3-vl-8b-instruct":{"country":"CN","id":"qwen3-vl-8b-instruct","launched":"2025-10-14","modalities":["visual"],"model":"Qwen3-VL 8B","model_family":"Qwen","model_variant":"Qwen3-VL-8B-Instruct","model_version":"Qwen3 VL","name":"Qwen3-VL-8B-Instruct","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3-vl-8b-thinking":{"country":"CN","id":"qwen3-vl-8b-thinking","launched":"2025-10-14","modalities":["visual"],"model":"Qwen3-VL 8B Thinking","model_family":"Qwen","model_variant":"Qwen3-VL-8B-Thinking","model_version":"Qwen3 VL","name":"Qwen3-VL-8B-Thinking","open_source":true,"parameter_size":"8B","provider":"qwen","url":""},"qwen3.5-122b-a10b":{"active_parameters":"10B","country":"CN","id":"qwen3.5-122b-a10b","launched":"2026-02-24","modalities":["text","visual"],"model":"Qwen3.5 122B A10B","model_family":"Qwen","model_variant":"Qwen3.5-122B-A10B","model_version":"Qwen3.5","name":"Qwen3.5-122B-A10B","open_source":true,"parameter_size":"122B","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3.5-122B-A10B"},"qwen3.5-27b":{"country":"CN","id":"qwen3.5-27b","launched":"2026-02-24","modalities":["text","visual"],"model":"Qwen3.5 27B","model_family":"Qwen","model_variant":"Qwen3.5-27B","model_version":"Qwen3.5","name":"Qwen3.5-27B","open_source":true,"parameter_size":"27B","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"qwen3.5-35b-a3b":{"active_parameters":"3B","country":"CN","id":"qwen3.5-35b-a3b","launched":"2026-02-24","modalities":["text","visual"],"model":"Qwen3.5 35B A3B","model_family":"Qwen","model_variant":"Qwen3.5-35B-A3B","model_version":"Qwen3.5","name":"Qwen3.5-35B-A3B","open_source":true,"parameter_size":"35B","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"qwen3.5-397b-a17b":{"active_parameters":"17B","country":"CN","id":"qwen3.5-397b-a17b","launched":"2026-02-16","modalities":["text","visual"],"model":"Qwen3.5 397B A17B","model_family":"Qwen","model_variant":"Qwen3.5-397B-A17B","model_version":"Qwen3.5","name":"Qwen3.5-397B-A17B","open_source":true,"parameter_size":"397B","provider":"qwen","url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"},"qwenlong-l1":{"active_parameters":"3B","country":"CN","id":"qwenlong-l1","launched":"2025-05-28","modalities":["text"],"model":"QwenLong-L1","model_family":"QwenLong","model_variant":"QwenLong-L1","model_version":"QwenLong","name":"QwenLong-L1","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwenlong-l1.5-30b-a3b":{"active_parameters":"3B","country":"CN","id":"qwenlong-l1.5-30b-a3b","launched":"2025-12-23","modalities":["text"],"model":"QwenLong-L1.5","model_family":"QwenLong","model_variant":"QwenLong-L1.5-30B-A3B","model_version":"QwenLong","name":"QwenLong-L1.5-30B-A3B","open_source":true,"parameter_size":"30B","provider":"qwen","url":""},"qwq-32b":{"country":"CN","id":"qwq-32b","launched":"2025-03-05","modalities":["text"],"model":"QwQ-32B","model_family":"QwQ","model_variant":"QwQ","model_version":"QwQ","name":"QwQ-32B","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"qwq-32b-plus-qwen2.5-coder-32b-instruct":{"country":"CN","id":"qwq-32b-plus-qwen2.5-coder-32b-instruct","launched":"","modalities":["text"],"model":"QwQ-32B + Qwen2.5 Coder 32B","model_family":"Composite","model_variant":"QwQ-32B + Qwen2.5 Coder 32B","model_version":"Composite","name":"QwQ-32B + Qwen2.5 Coder 32B Instruct","open_source":true,"parameter_size":"32B","provider":"ensemble","url":""},"qwq-32b-preview":{"country":"CN","id":"qwq-32b-preview","launched":"2025-03-05","modalities":["text"],"model":"QWQ 32B Preview","model_family":"QwQ","model_variant":"QWQ 32B Preview","model_version":"QwQ","name":"QWQ 32B Preview","open_source":true,"parameter_size":"32B","provider":"qwen","url":""},"r-4b-base":{"country":"CN","id":"r-4b-base","launched":"2025-08-20","modalities":["visual"],"model":"R 4B","model_family":"R","model_variant":"R-4b-base-4B","model_version":"R","name":"R-4B-Base","open_source":true,"parameter_size":"4B","provider":"tencent","url":""},"r-4b-rl":{"country":"CN","id":"r-4b-rl","launched":"2025-08-20","modalities":["visual"],"model":"R 4B","model_family":"R","model_variant":"R-4b-rl-4B","model_version":"R","name":"R-4B-RL","open_source":true,"parameter_size":"4B","provider":"tencent","url":""},"r2e-gym-32b":{"country":"US","id":"r2e-gym-32b","launched":"","modalities":["text"],"model":"R2E-Gym 32B","model_family":"R2E-Gym","model_variant":"32B","model_version":"R2E-Gym","name":"R2E-Gym-32B","open_source":true,"parameter_size":"32B","provider":"r2e-gym","url":""},"reflectioncoder-cl-34b":{"country":"US","id":"reflectioncoder-cl-34b","launched":"","modalities":["text"],"model":"Reflectioncoder CL 34B","model_family":"Reflectioncoder","model_variant":"Reflectioncoder CL 34B","model_version":"Reflectioncoder","name":"Reflectioncoder CL 34B","open_source":true,"parameter_size":"34B","provider":"reflection","url":""},"reflectioncoder-cl-7b":{"country":"US","id":"reflectioncoder-cl-7b","launched":"","modalities":["text"],"model":"Reflectioncoder CL 7B","model_family":"Reflectioncoder","model_variant":"Reflectioncoder CL 7B","model_version":"Reflectioncoder","name":"Reflectioncoder CL 7B","open_source":true,"parameter_size":"7B","provider":"reflection","url":""},"reflectioncoder-ds-33b":{"country":"US","id":"reflectioncoder-ds-33b","launched":"","modalities":["text"],"model":"Reflectioncoder DS 33B","model_family":"Reflectioncoder","model_variant":"Reflectioncoder DS 33B","model_version":"Reflectioncoder","name":"Reflectioncoder DS 33B","open_source":true,"parameter_size":"33B","provider":"reflection","url":""},"reflectioncoder-ds-6.7b":{"country":"US","id":"reflectioncoder-ds-6.7b","launched":"","modalities":["text"],"model":"Reflectioncoder DS 6.7B","model_family":"Reflectioncoder","model_variant":"Reflectioncoder DS 6.7B","model_version":"Reflectioncoder","name":"Reflectioncoder DS 6.7B","open_source":true,"parameter_size":"6.7B","provider":"reflection","url":""},"reka-core-20240904":{"country":"US","id":"reka-core-20240904","launched":"2024-09-04","modalities":["text"],"model":"Reka Core","model_family":"Reka","model_variant":"Reka Core 20240904","model_version":"Reka Core 20240904","name":"Reka Core 20240904","open_source":false,"parameter_size":"","provider":"reka","url":""},"reka-flash-20240904":{"country":"US","id":"reka-flash-20240904","launched":"2024-09-04","modalities":["text"],"model":"Reka Flash","model_family":"Reka","model_variant":"Reka Flash 20240904","model_version":"Reka Flash 20240904","name":"Reka Flash 20240904","open_source":false,"parameter_size":"","provider":"reka","url":""},"reka-flash-21b-20240226":{"country":"US","id":"reka-flash-21b-20240226","launched":"2024-02-26","modalities":["text"],"model":"Reka Flash 21B","model_family":"Reka","model_variant":"Reka Flash 21B 20240226","model_version":"Reka Flash 21B 20240226","name":"Reka Flash 21B 20240226","open_source":false,"parameter_size":"21B","provider":"reka","url":""},"reka-flash-21b-20240226-online":{"country":"US","id":"reka-flash-21b-20240226-online","launched":"2024-02-26","modalities":["text"],"model":"Reka Flash 21B","model_family":"Reka","model_variant":"Reka Flash 21B 20240226 Online","model_version":"Reka Flash 21B 20240226 Online","name":"Reka Flash 21B 20240226 Online","open_source":false,"parameter_size":"21B","provider":"reka","url":""},"reka-flash-3":{"country":"US","id":"reka-flash-3","launched":"","modalities":["text"],"model":"Reka Flash","model_family":"Reka Flash","model_variant":"Reka Flash 3","model_version":"Reka Flash","name":"Reka Flash 3","open_source":false,"parameter_size":"","provider":"reka","url":""},"ring-1t":{"country":"CN","id":"ring-1t","launched":"2025-10-13","modalities":["text"],"model":"Ring-1T","model_family":"Ring","model_variant":"Ring 1T","model_version":"Ring","name":"Ring-1T","open_source":true,"parameter_size":"1000B","provider":"antgroup","url":""},"ring-1t-2.5":{"country":"CN","id":"ring-1t-2.5","launched":"2026-02-12","modalities":["text"],"model":"Ring-1T-2.5","model_family":"Ring","model_variant":"Ring 1T 2.5","model_version":"Ring 2.5","name":"Ring-1T-2.5","open_source":true,"parameter_size":"1000B","provider":"antgroup","url":""},"ring-1t-2.5-heavy-thinking":{"country":"CN","id":"ring-1t-2.5-heavy-thinking","launched":"2026-02-12","modalities":["text"],"model":"Ring-1T-2.5-heavy-thinking","model_family":"Ring","model_variant":"Ring 1T 2.5 heavy thinking","model_version":"Ring 2.5","name":"Ring-1T-2.5-heavy-thinking","open_source":true,"parameter_size":"1000B","provider":"antgroup","url":""},"ring-1t-preview":{"country":"CN","id":"ring-1t-preview","launched":"2025-09-29","modalities":["text"],"model":"Ring-1T-preview","model_family":"Ring","model_variant":"1T-preview","model_version":"Ring","name":"Ring-1T-preview","open_source":true,"parameter_size":"","provider":"antgroup","url":""},"ring-flash-2.0":{"country":"CN","id":"ring-flash-2.0","launched":"2025-09-22","modalities":["text"],"model":"Ring Flash 2.0","model_family":"Ring","model_variant":"Flash 2.0","model_version":"Ring 2.0","name":"Ring Flash 2.0","open_source":true,"parameter_size":"103B","provider":"antgroup","url":""},"ring-flash-linear-2.0":{"country":"CN","id":"ring-flash-linear-2.0","launched":"2025-09-26","modalities":["text"],"model":"Ring Flash Linear 2.0","model_family":"Ring","model_variant":"Flash Linear 2.0","model_version":"Ring 2.0","name":"Ring Flash Linear 2.0","open_source":true,"parameter_size":"104B","provider":"antgroup","url":""},"ring-lite":{"country":"CN","id":"ring-lite","launched":"2025-06-17","modalities":["text"],"model":"Ring Lite","model_family":"Ring","model_variant":"Lite","model_version":"Ring Lite","name":"Ring-Lite","open_source":true,"parameter_size":"16.8B","provider":"antgroup","url":""},"ring-mini-2.0":{"active_parameters":"1.4B","country":"CN","id":"ring-mini-2.0","launched":"2025-09-08","modalities":["text"],"model":"Ring mini 2.0","model_family":"Ring","model_variant":"mini-2.0","model_version":"Ring 2.0","name":"Ring mini 2.0","open_source":true,"parameter_size":"16.8B","provider":"antgroup","url":""},"ring-mini-sparse-2.0-exp":{"country":"CN","id":"ring-mini-sparse-2.0-exp","launched":"2025-10-14","modalities":["text"],"model":"Ring mini sparse 2.0 exp","model_family":"Ring","model_variant":"mini-sparse-2.0-exp","model_version":"Ring 2.0","name":"Ring mini sparse 2.0 exp","open_source":true,"parameter_size":"16B","provider":"antgroup","url":""},"rnd1-base-0910":{"active_parameters":"3.3B","country":"JP","id":"rnd1-base-0910","launched":"2025-10-09","modalities":["text"],"model":"RND1 Base 0910","model_family":"RND1","model_variant":"RND1 Base 0910","model_version":"RND1","name":"RND1-Base-0910","open_source":true,"parameter_size":"30.5B","provider":"radicalnumerics","url":""},"rnj-1":{"country":"US","id":"rnj-1","launched":"2025-12-05","modalities":["text"],"model":"Rnj 1","model_family":"Rnj","model_variant":"1","model_version":"Rnj 1","name":"Rnj-1","open_source":true,"parameter_size":"8B","provider":"essential","url":""},"robobrain-2.0":{"country":"US","id":"robobrain-2.0","launched":"","modalities":["text"],"model":"RoboBrain 2.0","model_family":"RoboBrain","model_variant":"RoboBrain 2.0","model_version":"RoboBrain","name":"RoboBrain 2.0","open_source":false,"parameter_size":"","provider":"robobrain","url":""},"rwkv-4-raven-14b":{"country":"CN","id":"rwkv-4-raven-14b","launched":"","modalities":["text"],"model":"RWKV-4 Raven","model_family":"RWKV","model_variant":"RWKV-4-Raven-14B","model_version":"RWKV-4 Raven","name":"RWKV-4-Raven-14B","open_source":true,"parameter_size":"14B","provider":"rwkv","url":""},"rys-xlarge":{"country":"US","id":"rys-xlarge","launched":"","modalities":["text"],"model":"RYS XLarge","model_family":"RYS","model_variant":"RYS-Xlarge","model_version":"RYS","name":"RYS-Xlarge","open_source":false,"parameter_size":"78B","provider":"rys","url":""},"rys-xlarge-base":{"country":"US","id":"rys-xlarge-base","launched":"","modalities":["text"],"model":"RYS XLarge","model_family":"RYS","model_variant":"RYS-XLarge-Base","model_version":"RYS","name":"RYS-XLarge-Base","open_source":false,"parameter_size":"78B","provider":"rys","url":""},"s1.1-32b":{"country":"US","id":"s1.1-32b","launched":"","modalities":["text"],"model":"s1.1 32B","model_family":"s1","model_variant":"s1.1 32B","model_version":"s1","name":"s1.1 32B","open_source":true,"parameter_size":"32B","provider":"scale","url":""},"santacoder-1.1b":{"country":"FR","id":"santacoder-1.1b","launched":"","modalities":["text"],"model":"SantaCoder 1.1B","model_family":"SantaCoder","model_variant":"SantaCoder-1.1B","model_version":"SantaCoder","name":"SantaCoder-1.1B","open_source":true,"parameter_size":"1.1B","provider":"bigcode","url":""},"sd-xl":{"country":"US","id":"sd-xl","launched":"","modalities":["visual"],"model":"Stable Diffusion XL","model_family":"Stable Diffusion","model_variant":"XL","model_version":"SD XL","name":"SD XL","open_source":true,"parameter_size":"","provider":"stabilityai","url":""},"sd3-medium":{"country":"US","id":"sd3-medium","launched":"","modalities":["visual"],"model":"Stable Diffusion 3 Medium","model_family":"Stable Diffusion 3","model_variant":"Medium","model_version":"SD3","name":"SD3 Medium","open_source":true,"parameter_size":"","provider":"stabilityai","url":""},"seed-1.5-vl":{"active_parameters":"20B","country":"CN","id":"seed-1.5-vl","launched":"2025-05-13","modalities":["visual"],"model":"Seed 1.5 VL","model_family":"Seed","model_variant":"Seed-1.5-VL","model_version":"Seed 1.5","name":"Seed 1.5 VL","open_source":false,"parameter_size":"20B","provider":"bytedance","url":""},"seed-coder-8b-instruct":{"country":"CN","id":"seed-coder-8b-instruct","launched":"","modalities":["text"],"model":"Seed Coder 8B","model_family":"Seed","model_variant":"Seed Coder 8B Instruct","model_version":"Seed Coder","name":"Seed Coder 8B Instruct","open_source":true,"parameter_size":"8B","provider":"bytedance","url":""},"seed-oss-36b-instruct":{"country":"CN","id":"seed-oss-36b-instruct","launched":"2025-08-20","modalities":["text"],"model":"Seed-OSS 36B","model_family":"Seed","model_variant":"Seed-OSS-36B Instruct (think budget=0)","model_version":"Seed OSS","name":"Seed-OSS-36B-Instruct","open_source":true,"parameter_size":"36B","provider":"bytedance","url":""},"seed-thinking-v1.5":{"active_parameters":"20B","country":"CN","id":"seed-thinking-v1.5","launched":"2025-05-13","modalities":["text"],"model":"Seed Thinking v1.5","model_family":"Seed","model_variant":"Thinking v1.5","model_version":"Seed 1.5","name":"Seed Thinking v1.5","open_source":true,"parameter_size":"200B","provider":"bytedance","url":""},"seed1.5":{"country":"CN","id":"seed1.5","launched":"2025-05-13","modalities":["text"],"model":"Seed 1.5","model_family":"Seed","model_variant":"Seed1.5 Base","model_version":"Seed 1.5","name":"Seed1.5 Base","open_source":true,"parameter_size":"30Be","provider":"bytedance","url":""},"seed1.5-thinking":{"country":"CN","id":"seed1.5-thinking","launched":"2025-05-13","modalities":["text"],"model":"Seed 1.5","model_family":"Seed","model_variant":"Seed1.5","model_version":"Seed 1.5","name":"Seed1.5-Thinking","open_source":true,"parameter_size":"30Be","provider":"bytedance","url":""},"seed1.5-vl-thinking":{"country":"CN","id":"seed1.5-vl-thinking","launched":"2025-05-13","modalities":["visual"],"model":"Seed 1.5 VL","model_family":"Seed","model_variant":"Seed1.5-vl-thinking","model_version":"Seed 1.5","name":"Seed1.5-VL-Thinking","open_source":false,"parameter_size":"20Be","provider":"bytedance","url":""},"seed1.6":{"country":"CN","id":"seed1.6","launched":"2025-06-25","modalities":["text"],"model":"Seed 1.6","model_family":"Seed","model_variant":"Seed1.6","model_version":"Seed 1.6","name":"Seed1.6","open_source":true,"parameter_size":"50Be","provider":"bytedance","url":""},"seed1.6-thinking":{"country":"CN","id":"seed1.6-thinking","launched":"2025-06-25","modalities":["text"],"model":"Seed 1.6","model_family":"Seed","model_variant":"Seed1.6 Thinking","model_version":"Seed 1.6","name":"Seed1.6 Thinking","open_source":true,"parameter_size":"50Be","provider":"bytedance","url":""},"seed1.8":{"country":"CN","id":"seed1.8","launched":"2025-12-17","modalities":["text"],"model":"Seed 1.8","model_family":"Seed","model_variant":"Seed1.8","model_version":"Seed 1.8","name":"Seed1.8","open_source":false,"parameter_size":"","provider":"bytedance","url":""},"seed2.0-lite":{"country":"CN","id":"seed2.0-lite","launched":"2026-02-14","modalities":["text"],"model":"Seed 2.0 Lite","model_family":"Seed","model_variant":"Seed2.0 Lite","model_version":"Seed 2.0","name":"Seed2.0 Lite","open_source":false,"parameter_size":"","provider":"bytedance","url":""},"seed2.0-mini":{"country":"CN","id":"seed2.0-mini","launched":"2026-02-14","modalities":["text"],"model":"Seed 2.0 Mini","model_family":"Seed","model_variant":"Seed2.0 Mini","model_version":"Seed 2.0","name":"Seed2.0 Mini","open_source":false,"parameter_size":"","provider":"bytedance","url":""},"seed2.0-pro":{"country":"CN","id":"seed2.0-pro","launched":"2026-02-14","modalities":["text"],"model":"Seed 2.0 Pro","model_family":"Seed","model_variant":"Seed2.0 Pro","model_version":"Seed 2.0","name":"Seed2.0 Pro","open_source":false,"parameter_size":"","provider":"bytedance","url":""},"seedream-4.0":{"country":"CN","id":"seedream-4.0","launched":"2025-09-09","modalities":["visual"],"model":"Seedream 4.0","model_family":"Seedream","model_variant":"4.0","model_version":"Seedream","name":"Seedream 4.0","open_source":false,"parameter_size":"","provider":"bytedance","url":""},"seedream-4.5":{"country":"CN","id":"seedream-4.5","launched":"2025-12-03","modalities":["visual"],"model":"Seedream 4.5","model_family":"Seedream","model_variant":"4.5","model_version":"Seedream","name":"Seedream 4.5","open_source":false,"parameter_size":"","provider":"bytedance","url":""},"semcoder-6.7b":{"country":"US","id":"semcoder-6.7b","launched":"","modalities":["text"],"model":"SemCoder 6.7B","model_family":"SemCoder","model_variant":"SemCoder 6.7B","model_version":"SemCoder","name":"SemCoder 6.7B","open_source":true,"parameter_size":"6.7B","provider":"semcoder","url":""},"semcoder-6.7b-cot":{"country":"US","id":"semcoder-6.7b-cot","launched":"","modalities":["text"],"model":"SemCoder 6.7B CoT","model_family":"SemCoder","model_variant":"SemCoder 6.7B CoT","model_version":"SemCoder","name":"SemCoder 6.7B CoT","open_source":true,"parameter_size":"6.7B","provider":"semcoder","url":""},"semcoder-s-6.7b":{"country":"US","id":"semcoder-s-6.7b","launched":"","modalities":["text"],"model":"SemCoder-S 6.7B","model_family":"SemCoder","model_variant":"SemCoder-S 6.7B","model_version":"SemCoder","name":"SemCoder-S 6.7B","open_source":true,"parameter_size":"6.7B","provider":"semcoder","url":""},"semcoder-s-6.7b-cot":{"country":"US","id":"semcoder-s-6.7b-cot","launched":"","modalities":["text"],"model":"SemCoder-S 6.7B CoT","model_family":"SemCoder","model_variant":"SemCoder-S 6.7B CoT","model_version":"SemCoder","name":"SemCoder-S 6.7B CoT","open_source":true,"parameter_size":"6.7B","provider":"semcoder","url":""},"show-o2":{"country":"","id":"show-o2","launched":"","modalities":["visual"],"model":"Show-o2","model_family":"Show-o","model_variant":"o2","model_version":"Show-o2","name":"Show-o2","open_source":true,"parameter_size":"","provider":"unknown","url":""},"sky-t1-32b-flash":{"country":"US","id":"sky-t1-32b-flash","launched":"","modalities":["text"],"model":"SKY T1 32B Flash","model_family":"Sky","model_variant":"SKY T1 32B Flash","model_version":"Sky T1","name":"SKY T1 32B Flash","open_source":false,"parameter_size":"32B","provider":"novasky","url":""},"sky-t1-32b-preview":{"active_parameters":"32.0B","country":"US","id":"sky-t1-32b-preview","launched":"","modalities":["text"],"model":"Sky T1 32B","model_family":"Sky","model_variant":"Sky T1 32B Preview","model_version":"Sky T1","name":"Sky-T1-32B-Preview","open_source":false,"parameter_size":"32.0B","provider":"novasky","url":""},"skywork-or1-7b":{"country":"CN","id":"skywork-or1-7b","launched":"2025-05-13","modalities":["text"],"model":"Skywork OR1 7B","model_family":"Skywork OR1","model_variant":"7B","model_version":"Skywork OR1","name":"Skywork-OR1-7B","open_source":true,"parameter_size":"7B","provider":"KunLun","url":""},"skywork-r1v3-38b":{"country":"CN","id":"skywork-r1v3-38b","launched":"","modalities":["visual"],"model":"Skywork R1V3 38B","model_family":"Skywork","model_variant":"R1V3-38B","model_version":"Skywork R1V3","name":"Skywork-R1V3-38B","open_source":true,"parameter_size":"38B","provider":"skywork","url":""},"skywork-swe-32b":{"country":"CN","id":"skywork-swe-32b","launched":"","modalities":["text"],"model":"Skywork-SWE 32B","model_family":"Skywork","model_variant":"SWE 32B","model_version":"Skywork S