👩‍🔬 MaCBench Leaderboard

Can VLMs assist in scientific discovery?
Leaderboard Version
Dataset Version: 1.0.0
Leaderboard Version: 1.0.1
{
  • "headers": [
    • "Model",
    • "Open Source",
    • "Reasoning",
    • "Overall Score",
    • "afm-image",
    • "chem-lab-basic",
    • "chem-lab-comparison",
    • "chem-lab-equipments",
    • "chirality",
    • "cif-atomic-species",
    • "cif-crystal-system",
    • "cif-density",
    • "cif-symmetry",
    • "cif-volume",
    • "electronic-structure",
    • "handdrawn-molecules",
    • "isomers",
    • "mof-adsorption-strength-comparison",
    • "mof-adsorption-strength-order",
    • "mof-capacity-comparison",
    • "mof-capacity-order",
    • "mof-capacity-value",
    • "mof-henry-constant-comparison",
    • "mof-henry-constant-order",
    • "mof-working-capacity-comparison",
    • "mof-working-capacity-order",
    • "mof-working-capacity-value",
    • "org-schema-wo-smiles",
    • "org-schema",
    • "organic-molecules",
    • "spectral-analysis",
    • "tables-qa",
    • "us-patent-figures",
    • "us-patent-plots",
    • "xrd-pattern-matching",
    • "xrd-pattern-shape",
    • "xrd-peak-position",
    • "xrd-relative-intensity"
    ],
  • "data": [
    • [
      • "Claude-3.5-Sonnet",
      • false,
      • false,
      • 0.67,
      • 0.43,
      • 0.28,
      • 0.49,
      • 0.84,
      • 0.66,
      • 0.95,
      • 0.53,
      • 0.39,
      • 0.6,
      • 0.96,
      • 0.7,
      • 0.97,
      • 0.3,
      • 0.93,
      • 0.74,
      • 0.99,
      • 0.85,
      • 0.74,
      • 1,
      • 0.82,
      • 0.76,
      • 0.71,
      • 0.67,
      • 0.76,
      • 1,
      • 0.8,
      • 0.28,
      • 0.67,
      • 0.67,
      • 0.64,
      • 0.52,
      • 0.89,
      • 1,
      • 0.46
      ],
    • [
      • "Gemini-1.5-Pro",
      • false,
      • false,
      • 0.57,
      • 0.21,
      • 0.55,
      • 0.41,
      • 0.84,
      • 0.44,
      • 0.81,
      • 0.69,
      • 0.4,
      • 0.66,
      • 0.96,
      • 0.39,
      • 0.97,
      • 0.25,
      • 0.68,
      • 0.49,
      • 0.99,
      • 0.76,
      • 0.65,
      • 0.64,
      • 0.67,
      • 0.55,
      • 0.5,
      • 0.33,
      • 0.85,
      • 1,
      • 0.59,
      • 0.35,
      • 0.61,
      • 0.32,
      • 0.22,
      • 0.27,
      • 0.71,
      • 0.85,
      • 0.35
      ],
    • [
      • "GPT-4o",
      • false,
      • false,
      • 0.54,
      • 0.21,
      • 0.25,
      • 0.47,
      • 0.87,
      • 0.5,
      • 0.82,
      • 0.57,
      • 0.31,
      • 0.28,
      • 0.83,
      • 0.56,
      • 0.63,
      • 0.25,
      • 0.6,
      • 0.78,
      • 0.88,
      • 0.63,
      • 0.44,
      • 0.88,
      • 0.75,
      • 0.53,
      • 0.64,
      • 0.24,
      • 0.73,
      • 1,
      • 0.56,
      • 0.43,
      • 0.54,
      • 0.54,
      • 0.51,
      • 0.28,
      • 0.85,
      • 0.8,
      • 0.17
      ],
    • [
      • "Llama 3.2 90B Vision",
      • true,
      • false,
      • 0.36,
      • 0.14,
      • 0.24,
      • 0.47,
      • 0.53,
      • 0.28,
      • 0.83,
      • 0.33,
      • 0.21,
      • 0.24,
      • 0.43,
      • 0.39,
      • 0.72,
      • 0.15,
      • 0.07,
      • 0.37,
      • 0.63,
      • 0.55,
      • 0.55,
      • 0.8,
      • 0.75,
      • 0.45,
      • 0.55,
      • 0.25,
      • 0.59,
      • 0.5,
      • 0.53,
      • 0.35,
      • 0.31,
      • 0.27,
      • 0.28,
      • 0.45,
      • 0.3,
      • 0.3,
      • 0.16
      ],
    • [
      • "llama-3.2-11b-vision-preview",
      • true,
      • false,
      • 0.32,
      • 0.14,
      • 0.32,
      • 0.29,
      • 0.64,
      • 0.48,
      • 0.85,
      • 0.19,
      • 0.02,
      • 0.5,
      • 0.14,
      • 0.43,
      • 0.66,
      • 0.3,
      • 0.27,
      • 0.26,
      • 0.21,
      • 0.35,
      • 0.25,
      • 0.3,
      • 0.33,
      • 0.25,
      • 0.25,
      • 0.15,
      • 0.59,
      • 0.25,
      • 0.53,
      • 0.45,
      • 0.27,
      • 0.3,
      • 0.31,
      • 0.65,
      • 0.15,
      • 0.3,
      • 0.21
      ],
    • [
      • "JanusPro-7B",
      • true,
      • false,
      • 0.2,
      • 0.1,
      • 0.03,
      • 0.06,
      • 0.04,
      • 0.08,
      • 0.46,
      • 0.07,
      • 0,
      • 0.02,
      • 0.81,
      • 0,
      • 0.03,
      • 0,
      • 0.13,
      • 0,
      • 0.32,
      • 0,
      • 0.25,
      • 0.1,
      • 0,
      • 0.2,
      • 0,
      • 0.2,
      • 0.18,
      • 0.25,
      • 0.2,
      • 0.15,
      • 0.28,
      • 0.46,
      • 0.25,
      • 0,
      • 0,
      • 0.2,
      • 0
      ]
    ],
  • "metadata": null
}