{
  "schema_version": 1,
  "recorded_at": "2026-08-11",
  "ship": {
    "repository": "toolboxmd/karpathy-wiki",
    "implementation_commit": "877e659",
    "implementation_commit_url": "https://github.com/toolboxmd/karpathy-wiki/commit/877e659",
    "baseline_commit": "990cb20",
    "full_test_suite": {
      "passed": 90,
      "failed": 0
    }
  },
  "benchmark": {
    "skill_snapshot_commit": "990cb20",
    "skill_snapshot_version": "0.2.8",
    "model": "gpt-5.3-codex-spark",
    "protocol": "One independent run per reasoning effort, with cold-start, exact-duplicate, and related-augmentation cases executed sequentially inside each run.",
    "retrieval_questions_per_candidate": 8,
    "blind_judge_read_authored_wiki_first": true,
    "single_run_per_configuration": true,
    "results": [
      {
        "reasoning_effort": "low",
        "blind_candidate": "candidate-001",
        "deterministic_assertions_passed": 16,
        "deterministic_assertions_total": 21,
        "blind_semantic_score": 55,
        "retrieval_full": 1,
        "retrieval_partial": 6,
        "retrieval_none": 1,
        "duration_seconds": 766,
        "terminal_wiki_state_clean": false
      },
      {
        "reasoning_effort": "medium",
        "blind_candidate": "candidate-003",
        "deterministic_assertions_passed": 19,
        "deterministic_assertions_total": 21,
        "blind_semantic_score": 69,
        "retrieval_full": 3,
        "retrieval_partial": 5,
        "retrieval_none": 0,
        "duration_seconds": 480,
        "terminal_wiki_state_clean": true
      },
      {
        "reasoning_effort": "high",
        "blind_candidate": "candidate-002",
        "deterministic_assertions_passed": 17,
        "deterministic_assertions_total": 21,
        "blind_semantic_score": 54,
        "retrieval_full": 1,
        "retrieval_partial": 7,
        "retrieval_none": 0,
        "duration_seconds": 737,
        "terminal_wiki_state_clean": false
      }
    ],
    "interpretation_limit": "The comparison is directional, not a variance estimate. Replicated runs are required before treating a small score difference as stable.",
    "artifact_sha256": {
      "blind_evaluation_json": "33df4f97d02473a68b529893f8f27a2cee6508936badcf47a65a40f8a9d2ef7e",
      "deterministic_results_json": "eadeb76570cc464352272358dfc1ef28f55f18d04ce363d8de3730aa20e1069d",
      "model_attribution_audit_json": "7735ef743fde3656146ece883860ab0b8c9e20c7fbce2126937b28aeb0a1a2b0",
      "read_isolation_audit_json": "5e32d1c8e0037fb955bef3c16e2a3784b4f76e8bc95ca5949561960fe95ecc27",
      "protocol_amendment_md": "be7d44a42fdefba94f88c3e5102350ca8ceb398a4c900af15bebd5b38eea2e84"
    }
  },
  "invalid_attempts": [
    {
      "status": "invalid_harness_contamination",
      "reasoning_effort": "low",
      "completed_cases_before_stop": 1,
      "active_case_at_stop": "exact-duplicate",
      "failure": "The model under test invoked a nested Claude ingester, so the resulting wiki could not be attributed solely to the requested model.",
      "scoring_eligible": false
    },
    {
      "status": "invalid_cross_run_read_leakage",
      "reasoning_effort": "medium",
      "completed_cases_before_stop": 0,
      "active_case_at_stop": "cold-start",
      "failure": "The model read global memory, sibling benchmark output, and grader source before its first wiki write.",
      "scoring_eligible": false
    }
  ],
  "accepted_protocol_amendments": [
    "The model under test must execute the skill itself and may not delegate to another model, agent, or agentic CLI.",
    "A run may read only its own wiki, capture, named evidence, frozen skill, frozen plugin, and files directly referenced by the frozen skill.",
    "Candidate identity remains outside the blind-review tree; the blind evaluator reads authored wiki output before deterministic evidence."
  ],
  "runtime_acceptance": {
    "provider": "codex",
    "model": "gpt-5.3-codex-spark",
    "reasoning_effort": "medium",
    "cases": [
      "cold ingest",
      "exact duplicate",
      "related augmentation"
    ],
    "codexbar_available": false,
    "nested_agentic_invocations": 0,
    "final_slot_or_processing_leaks": 0,
    "additional_real_harness_checks": [
      "Clean SessionStart mode",
      "Clean scheduled mode",
      "Temporary macOS LaunchAgent install, run, and uninstall"
    ]
  }
}