{
  "kind": "route",
  "format": "kosplora-route/0.2",
  "question": "Can you write an AI evaluation that someone else can rerun?",
  "move": [
    "practice",
    "make"
  ],
  "audience": "Some Python. Runs on your own computer against a model you run yourself, so no API key or payment is needed.",
  "time": {
    "solo": "4h",
    "together": "45m"
  },
  "authors": [
    {
      "name": "Bentley Moon",
      "date": "2026-10-21"
    }
  ],
  "would_change": "If Inspect stopped supporting models run on your own computer, station 3 would move to whichever local option it supports then. If BetterBench revised its criteria, the self-assessment would follow the new list, dated.",
  "standard": {
    "name": "BetterBench: the benchmark quality criteria and checklist",
    "publisher": "Anka Reuel, Amelia Hardy, Chandler Smith, Max Lamparth, Malcolm Hardy and Mykel J. Kochenderfer, Stanford University",
    "date": "2024-11-20",
    "url": "https://betterbench.stanford.edu/checklist.html",
    "meets": "The evaluation states what it tests and why, ships its code and requirements, and documents how to rerun it, as the checklist's design, implementation and documentation criteria ask. A second person reruns it from your files alone and gets the same scores on the same model.",
    "checks": [],
    "assessed_by": "You, criterion by criterion against the checklist, and then one other person who reruns the evaluation from your files without your help."
  },
  "stations": [
    {
      "type": "orient",
      "time": "40m",
      "task": "Work through the tutorial of the UK AI Security Institute's open-source evaluation framework. Write down the three parts every evaluation has in it: the dataset, the solver and the scorer.",
      "sources": [
        {
          "title": "Inspect tutorial",
          "author": "UK AI Security Institute",
          "date": "",
          "url": "https://inspect.aisi.org.uk/tutorial.html",
          "origin": "published"
        }
      ]
    },
    {
      "type": "source",
      "time": "40m",
      "task": "Read the summary of the paper and the criteria on the methodology page. Pick the five criteria you think a small evaluation most often fails, and say why.",
      "sources": [
        {
          "title": "BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices",
          "author": "Anka Reuel, Amelia Hardy, Chandler Smith, Max Lamparth, Malcolm Hardy and Mykel J. Kochenderfer",
          "date": "2024-11-20",
          "url": "https://arxiv.org/abs/2411.12990",
          "origin": "published"
        }
      ]
    },
    {
      "type": "practice",
      "time": "45m",
      "task": "Install Inspect and a model you can run on your own computer through Ollama, then run one of the tutorial's evaluations against it, following the model providers page. Write down the command, the model and its version, and the score.",
      "sources": [
        {
          "title": "Inspect: model providers, Ollama",
          "author": "UK AI Security Institute",
          "date": "",
          "url": "https://inspect.aisi.org.uk/providers.html",
          "origin": "published"
        }
      ],
      "alternatives": [
        "If your computer cannot run a model, read through the tutorial's logs instead and write the evaluation in station 5 without running it; the second person in station 6 runs it."
      ]
    },
    {
      "type": "counterpoint",
      "time": "30m",
      "task": "Read the summary of a paper arguing that benchmarks are often presented as measuring far more than they can. Write down, in one sentence, the narrowest honest claim your evaluation could support.",
      "sources": [
        {
          "title": "AI and the Everything in the Whole Wide World Benchmark",
          "author": "Inioluwa Deborah Raji, Emily M. Bender, Amandalynne Paullada, Emily Denton and Alex Hanna",
          "date": "2021-11-26",
          "url": "https://arxiv.org/abs/2111.15366",
          "origin": "published"
        }
      ]
    },
    {
      "type": "make",
      "time": "60m",
      "task": "Write a small evaluation of your own in Inspect: ten to twenty samples that test one stated thing, a scorer, a requirements file, and a short README saying what it tests, what it does not, and the exact command to rerun it. The nine texts from the route on hidden instructions make a good dataset. Score it against the checklist's design, implementation and documentation criteria."
    },
    {
      "type": "person",
      "time": "15m",
      "task": "Give your files to one other person and ask them to rerun the evaluation on the same model without asking you anything. Note whether they could, and whether the scores matched."
    },
    {
      "type": "reflect",
      "time": "10m",
      "task": "Record what the rerun taught you that the checklist did not. If you want to go further, chapter 3 of the course below covers evaluations of language models in depth.",
      "sources": [
        {
          "title": "ARENA course materials, chapter 3: LLM evaluations",
          "author": "ARENA",
          "date": "",
          "url": "https://learn.arena.education",
          "origin": "published"
        }
      ]
    }
  ],
  "artifact": {
    "keeps": "A small evaluation in Inspect with its code, requirements and README, scored against BetterBench's criteria, and a record of whether a second person could rerun it and got the same scores."
  },
  "exit": [
    "keep",
    "share",
    "fork",
    "correct-a-source",
    "close"
  ],
  "revisions": [
    {
      "date": "2026-10-21",
      "change": "First version.",
      "by": "Bentley Moon"
    }
  ]
}