{
  "kind": "route",
  "format": "kosplora-route/0.2",
  "question": "Can you reproduce a published AI safety result yourself?",
  "move": [
    "understand",
    "practice"
  ],
  "audience": "Some comfort running a command in a terminal. No programming or API key needed.",
  "time": {
    "solo": "2h30m",
    "together": "45m"
  },
  "authors": [
    {
      "name": "Bentley Moon",
      "date": "2026-10-20"
    }
  ],
  "would_change": "A rerun of the release that does not print 0.80 and 1.00 for Claude Sonnet 5 would restructure the route: the finished work would become a written discrepancy report sent to the author, and station 3 would add a check of the environment before the run.",
  "standard": {
    "name": "ACM Artifact Review and Badging, version 1.1: Results Reproduced",
    "publisher": "Association for Computing Machinery",
    "date": "2020-08-24",
    "url": "https://www.acm.org/publications/policies/artifact-review-and-badging-current",
    "meets": "Someone other than the authors obtains the central result, using at least in part the authors' own artifacts.",
    "checks": [
      {
        "label": "Claude Sonnet 5, share of tries where it obeys an instruction placed last in the prompt (64 of 80)",
        "expected": "0.80"
      },
      {
        "label": "Claude Sonnet 5, share of injections it detects when asked",
        "expected": "1.00"
      }
    ],
    "assessed_by": "You, against the published table, and then one other person who reruns one number."
  },
  "stations": [
    {
      "type": "orient",
      "time": "25m",
      "task": "Read the release's summary of the result: one model detects every injected instruction when asked to audit, and still obeys it on 64 of 80 tries when it is not asked, as long as the instruction is the last text in the prompt.",
      "sources": [
        {
          "title": "Detection is not resistance: frontier prompt-injection receipts, version 1.1.0",
          "author": "Bentley Moon",
          "date": "2026-09-22",
          "url": "https://doi.org/10.5281/zenodo.22805797",
          "origin": "dataset"
        }
      ]
    },
    {
      "type": "source",
      "time": "15m",
      "task": "Read what counts as a reproduced result, and how it differs from a replicated one.",
      "sources": [
        {
          "title": "Artifact Review and Badging, version 1.1",
          "author": "Association for Computing Machinery",
          "date": "2020-08-24",
          "url": "https://www.acm.org/publications/policies/artifact-review-and-badging-current",
          "origin": "published"
        }
      ]
    },
    {
      "type": "practice",
      "time": "30m",
      "task": "Download the release from its DOI, open a terminal in its folder and run python reproduce.py. It needs no network, no API key and no extra packages. Write down the obey and detect figures it prints for each model.",
      "alternatives": [
        "If you cannot run Python, open the data folder in the release and recount the Sonnet 5 rows by hand: the script reads the same per-task values."
      ]
    },
    {
      "type": "counterpoint",
      "time": "30m",
      "task": "Read an independent account of prompt injection and write down one way a different test would change the 0.80: where the injected text sits, what the model was asked to do, or how obeying was judged.",
      "sources": [
        {
          "title": "Not what you've signed up for: compromising real-world LLM-integrated applications with indirect prompt injection",
          "author": "Kai Greshake, Sahar Abdelnabi, Shailesh Mishra, Christoph Endres, Thorsten Holz and Mario Fritz",
          "date": "2023-02-23",
          "url": "https://arxiv.org/abs/2302.12173",
          "origin": "published"
        }
      ]
    },
    {
      "type": "make",
      "time": "30m",
      "task": "Write a one-page reproduction report: the release version, your computer and Python version, the command you ran, the figures you got, whether they match the published table, and anything that differed."
    },
    {
      "type": "person",
      "time": "15m",
      "task": "Give the report and the release to one other person and ask them to rerun one number. Note whether they got the same figure."
    },
    {
      "type": "reflect",
      "time": "10m",
      "task": "Record what the result does and does not show, in your own words, and one question it leaves open."
    }
  ],
  "artifact": {
    "keeps": "A one-page reproduction report of a published AI safety result, with the figures you obtained, a second person's rerun, and the standard it was checked against."
  },
  "exit": [
    "keep",
    "share",
    "fork",
    "correct-a-source",
    "close"
  ],
  "revisions": [
    {
      "date": "2026-10-20",
      "change": "First version.",
      "by": "Bentley Moon"
    }
  ]
}