{
  "metric": "soft_accuracy",
  "metric_direction": "higher",
  "baseline": "jev-1.13.0",
  "runs": [
    "jev-1.13.0",
    "bonsai-2-27b-calibrated",
    "bonsai-2-27b-calibration-init",
    "laya",
    "openjev-qwen3.5-4b",
    "spark-X2.5",
    "von-1.1"
  ],
  "common_capabilities": 275,
  "union_capabilities": 275,
  "summary": [
    {
      "run": "jev-1.13.0",
      "capabilities_present": 275,
      "common_capabilities": 275,
      "coverage": 1.0,
      "macro": 0.8808149090909092,
      "weighted": 0.8808037771010409,
      "median": 0.9241,
      "p10": 0.69558,
      "examples": 22001,
      "min_n": 80,
      "reported_micro": 0.8808,
      "reported_micro_n": 22001,
      "macro_accuracy": 0.9340567272727271,
      "macro_soft_accuracy": 0.8808149090909092,
      "macro_nll": 0.22846981818181816,
      "macro_ece_15": 0.08400436363636363,
      "macro_brier": 0.1070130909090909,
      "macro_latency_ms_p50": 716.3643636363637,
      "macro_latency_ms_p95": 778.8174545454546,
      "macro_error_rate": 0.0
    },
    {
      "run": "bonsai-2-27b-calibrated",
      "capabilities_present": 275,
      "common_capabilities": 275,
      "coverage": 1.0,
      "macro": 0.7646050909090909,
      "weighted": 0.7645855370210444,
      "median": 0.7934,
      "p10": 0.49522,
      "examples": 22001,
      "min_n": 80,
      "reported_micro": 0.7646,
      "reported_micro_n": 22001,
      "macro_accuracy": 0.833032,
      "macro_soft_accuracy": 0.7646050909090909,
      "macro_nll": 0.43494327272727273,
      "macro_ece_15": 0.13046036363636362,
      "macro_brier": 0.23390981818181816,
      "macro_latency_ms_p50": 170.93018181818184,
      "macro_latency_ms_p95": 448.0192727272728,
      "macro_error_rate": 0.0
    },
    {
      "run": "bonsai-2-27b-calibration-init",
      "capabilities_present": 275,
      "common_capabilities": 275,
      "coverage": 1.0,
      "macro": 0.762941818181818,
      "weighted": 0.7629224307986001,
      "median": 0.7984,
      "p10": 0.49872,
      "examples": 22001,
      "min_n": 80,
      "reported_micro": 0.7629,
      "reported_micro_n": 22001,
      "macro_accuracy": 0.8329865454545454,
      "macro_soft_accuracy": 0.762941818181818,
      "macro_nll": 0.43797709090909087,
      "macro_ece_15": 0.1336378181818182,
      "macro_brier": 0.2354927272727273,
      "macro_latency_ms_p50": 164.9741818181818,
      "macro_latency_ms_p95": 362.0116363636364,
      "macro_error_rate": 0.0
    },
    {
      "run": "laya",
      "capabilities_present": 275,
      "common_capabilities": 275,
      "coverage": 1.0,
      "macro": 0.4669730909090909,
      "weighted": 0.4669658515522021,
      "median": 0.4554,
      "p10": 0.25232000000000004,
      "examples": 22001,
      "min_n": 80,
      "reported_micro": 0.467,
      "reported_micro_n": 22001,
      "macro_accuracy": 0.5254803636363635,
      "macro_soft_accuracy": 0.4669730909090909,
      "macro_nll": 1.248675272727273,
      "macro_ece_15": 0.2779410909090909,
      "macro_brier": 0.6382174545454546,
      "macro_latency_ms_p50": 36.72290909090909,
      "macro_latency_ms_p95": 44.35090909090909,
      "macro_error_rate": 0.0
    },
    {
      "run": "openjev-qwen3.5-4b",
      "capabilities_present": 275,
      "common_capabilities": 275,
      "coverage": 1.0,
      "macro": 0.7413254545454545,
      "weighted": 0.7413146356983773,
      "median": 0.7691,
      "p10": 0.44432,
      "examples": 22001,
      "min_n": 80,
      "reported_micro": 0.7413,
      "reported_micro_n": 22001,
      "macro_accuracy": 0.7911505454545454,
      "macro_soft_accuracy": 0.7413254545454545,
      "macro_nll": 0.5495214545454545,
      "macro_ece_15": 0.14347818181818184,
      "macro_brier": 0.29464690909090907,
      "macro_latency_ms_p50": 1066.06,
      "macro_latency_ms_p95": 1478.8843636363638,
      "macro_error_rate": 0.0
    },
    {
      "run": "spark-X2.5",
      "capabilities_present": 275,
      "common_capabilities": 275,
      "coverage": 1.0,
      "macro": 0.7096698181818182,
      "weighted": 0.7096582246261535,
      "median": 0.7368,
      "p10": 0.41892,
      "examples": 22001,
      "min_n": 80,
      "reported_micro": 0.7097,
      "reported_micro_n": 22001,
      "macro_accuracy": 0.714248,
      "macro_soft_accuracy": 0.7096698181818182,
      "macro_nll": 1.745493090909091,
      "macro_ece_15": 0.2494050909090909,
      "macro_brier": 0.5012818181818183,
      "macro_latency_ms_p50": 1056.5669090909091,
      "macro_latency_ms_p95": 1783.474909090909,
      "macro_error_rate": 0.0
    },
    {
      "run": "von-1.1",
      "capabilities_present": 275,
      "common_capabilities": 275,
      "coverage": 1.0,
      "macro": 0.48573818181818185,
      "weighted": 0.4857248715967456,
      "median": 0.4277,
      "p10": 0.21956,
      "examples": 22001,
      "min_n": 80,
      "reported_micro": 0.4857,
      "reported_micro_n": 22001,
      "macro_accuracy": 0.49580872727272723,
      "macro_soft_accuracy": 0.48573818181818185,
      "macro_nll": 2.7344818181818185,
      "macro_ece_15": 0.37165200000000004,
      "macro_brier": 0.8136847272727272,
      "macro_latency_ms_p50": 38.470181818181814,
      "macro_latency_ms_p95": 46.46509090909091,
      "macro_error_rate": 0.0
    }
  ]
}