{
  "release_id": "SC-EVAL-2026-08-06",
  "status": "approved-for-publication",
  "published_at_utc": "2026-08-06",
  "scope": "Primary general-purpose CPU inference profile on Genie 1 and Genie 2",
  "capability": {
    "benchmark": "MMLU-Pro-280 balanced direct-answer audit",
    "metric": "Exact-match accuracy",
    "best_observed": {
      "server": "Genie 2",
      "score_percent": 64.64,
      "correct": 181,
      "total": 280,
      "reproduction_score_percent": 64.29,
      "reproduction_correct": 180,
      "spread_percentage_points": 0.35
    },
    "servers": [
      {
        "server": "Genie 1",
        "model": "Superchat Genie 1 production build",
        "run_1": { "score_percent": 58.21, "correct": 163, "total": 280 },
        "run_2": { "score_percent": 58.21, "correct": 163, "total": 280 },
        "spread_percentage_points": 0.0,
        "unknown": 0,
        "errors": 0
      },
      {
        "server": "Genie 2",
        "model": "Superchat Genie 2 production build",
        "run_1": { "score_percent": 64.64, "correct": 181, "total": 280 },
        "run_2": { "score_percent": 64.29, "correct": 180, "total": 280 },
        "spread_percentage_points": 0.35,
        "unknown": 0,
        "errors": 0
      }
    ],
    "conditions": {
      "dataset": "TIGER-Lab/MMLU-Pro test split",
      "dataset_parquet_sha256": "0e24a191921c2f453518a537a8b2117bd137e7714d4ef1565e9ba06c1ecb9ad8",
      "upstream_repository_commit": "f418b116db00b065c2aea046518d8fcf74d39872",
      "sample": "20 questions from each of 14 domains, selected by the lowest SHA-256(seed:question_id) values",
      "sample_size": 280,
      "sample_seed": 20260806,
      "sample_file_sha256": "514a61884015867a266efe467dbd77a1a8833c24f56d855cc1244b5a64c12c24",
      "prompting": "Zero-shot direct answer; one option letter; chain of thought disabled",
      "temperature": 0,
      "top_p": 1,
      "request_seed": 20260806,
      "max_output_tokens": 16,
      "concurrency": 1,
      "evaluator": "benchmarks/run_mmlu_pro_audit.py 1.0.1",
      "retry_policy": "One retry permitted for transport or JSON errors only; no wrong answer was retried",
      "limitations": "This is a balanced audit derived from MMLU-Pro. It is not the full 12,032-question, five-shot chain-of-thought MMLU-Pro leaderboard protocol and must not be compared as an official leaderboard submission."
    }
  },
  "serving": {
    "benchmark": "Superchat fixed single-stream decode test",
    "metric": "Median server-reported decode tokens per second",
    "best_observed": {
      "server": "Genie 1",
      "tokens_per_second": 23.06
    },
    "servers": [
      {
        "server": "Genie 1",
        "median_decode_tokens_per_second": 23.06,
        "mean_decode_tokens_per_second": 23.17,
        "median_prompt_tokens_per_second": 119.35,
        "measured_runs": 5
      },
      {
        "server": "Genie 2",
        "median_decode_tokens_per_second": 15.66,
        "mean_decode_tokens_per_second": 15.62,
        "median_prompt_tokens_per_second": 86.09,
        "measured_runs": 5
      }
    ],
    "conditions": {
      "warmup_runs": 1,
      "measured_runs": 5,
      "generated_tokens_per_run": 256,
      "concurrency": 1,
      "streaming": false,
      "thinking": false,
      "temperature": 0,
      "score": "Median server-reported decode rate across five fixed prompts"
    }
  },
  "hardware": [
    {
      "server": "Genie 1",
      "cpu": "1 x AMD EPYC 7502P, 32 cores / 64 threads",
      "cache": "128 MiB L3",
      "numa_nodes": 1,
      "memory": "128 GB installed (4 x 32 GB), configured at 3200 MT/s",
      "operating_system": "Linux 6.1.0-49-amd64 x86_64",
      "model_file_size_bytes": 27087812896,
      "model_weights_sha256": "8ca8224c5519b50f286b9739f9ab1c2bc0759e02cfb6cbf48a09f0759eb2d319",
      "inference_engine": "ik_llama.cpp",
      "inference_engine_commit": "bbc7de475178dd0535c16ad85f204a2529806c9d",
      "server_profile": "16 inference threads, 24,576-token context, one slot, flash attention, MTP speculative decoding"
    },
    {
      "server": "Genie 2",
      "cpu": "1 x AMD EPYC 7401P, 24 cores / 48 threads",
      "cache": "64 MiB L3",
      "numa_nodes": 4,
      "memory": "256 GB installed (8 x 32 GB), modules rated 2667 MT/s and configured at 2400 MT/s",
      "operating_system": "Linux 6.1.0-49-amd64 x86_64",
      "model_file_size_bytes": 21314116128,
      "model_weights_sha256": "7f6180b97db29506386cc9e50ce6baba336b168d1fbbc161c73529a3e6e5ee5b",
      "inference_engine": "ik_llama.cpp",
      "inference_engine_commit": "31018dc51135a8a3ded085fa7e198befff19ebf4",
      "server_profile": "24 inference threads, 32,768-token total context / four slots, 8,192 tokens per slot, flash attention"
    }
  ],
  "artifacts": {
    "run_1": "benchmarks/results/run-1/",
    "run_2": "benchmarks/results/run-2/",
    "genie_1_decode": "benchmarks/results/genie-1-decode-summary.json",
    "genie_2_decode": "benchmarks/results/genie-2-decode-summary.json",
    "harness": "benchmarks/run_mmlu_pro_audit.py"
  }
}