{
 "body": "\nPart of the [BIG-bench](big_bench.md) family.\n\n## What it measures\n\nModified Arithmetic tests whether a model infers a new operation from five examples. In the challenge form, the model must perform addition, subtraction, or multiplication and then add one. Control forms use the ordinary operation with the same number format.\n\n## Reading the numbers\n\nThe task uses exact string matching on the numerical answer. A high score on a +1 subtask suggests the model followed the demonstrated rule rather than applying a memorized arithmetic pattern. The README reports GPT-3 at 0% on all three +1 subtasks and nonzero control results, but those small reported experiments are not a current leaderboard.\n\nThe five demonstrations are regenerated for each question, so memorizing one worked prompt does not solve the collection. Control and +1 variants should be read as a paired comparison. Long explanations or mathematically equivalent formatting can still fail exact string matching.\n",
 "build": {
  "built_at": "2026-09-09T16:56:50+00:00",
  "commit": "0a599558854c0e238c03a0f0d725239cb28f9d11",
  "eligibility_as_of": "2026-09-09"
 },
 "disposition": {
  "canonical_id": "modified_arithmetic",
  "reasons": [],
  "status": "unassessed",
  "verified_results": []
 },
 "models_covered": [],
 "page": {
  "aliases": [
   "BIG-bench modified_arithmetic"
  ],
  "category": "math",
  "contamination": {
   "note": "The task is public and fixed, with a BIG-bench canary. Random examples in prompts vary, but no contamination study was opened.",
   "risk": "medium"
  },
  "dataset": {
   "languages": [
    "en"
   ],
   "license": "Apache-2.0",
   "modalities": [
    "text"
   ],
   "public_test_set": true,
   "size": 6000,
   "size_note": "The task README states 1,000 questions for each of six subtasks.",
   "splits": "Six fixed subtask collections; no train/test split",
   "url": "https://github.com/google/BIG-bench/tree/main/bigbench/benchmark_tasks/modified_arithmetic"
  },
  "freshness": {
   "researched": "2026-09-08",
   "researched_by": "GPT-5.6 Luna, luna-batch-059 (Codex coordinated)",
   "reviewed": "2026-09-08",
   "reviewed_by": "Claude Sonnet 5 independent review, luna-batch-059"
  },
  "harness": {
   "bigbench": "modified_arithmetic",
   "helm": "",
   "inspect_evals": "",
   "lm_eval": "",
   "opencompass": "",
   "other": ""
  },
  "id": "modified_arithmetic",
  "last_updated": "",
  "leaderboard_url": "",
  "lineage": {
   "family": "big_bench",
   "predecessor": "",
   "successors": [],
   "variants": []
  },
  "measures": "Modified Arithmetic gives two numbers and five worked examples using an operation. The model must complete a sixth example. In the challenge subtasks the ordinary operation is followed by adding one, while control subtasks omit the extra one.\n",
  "metric": {
   "baseline_note": "The output space is numerical and varies by prompt, so a uniform random baseline is not established.",
   "direction": "higher_is_better",
   "human_baseline": null,
   "max_score": 100,
   "name": "exact_str_match",
   "random_baseline": null,
   "unit": "%"
  },
  "name": "Modified Arithmetic",
  "page_kind": "subset",
  "paper": {
   "arxiv": "2206.04615",
   "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
   "url": "https://arxiv.org/abs/2206.04615",
   "year": 2022
  },
  "publisher": {
   "authors": [
    "Jack Geissinger",
    "James Simon"
   ],
   "org": "Google BIG-bench",
   "url": "https://github.com/google/BIG-bench/tree/main/bigbench/benchmark_tasks/modified_arithmetic"
  },
  "released": "2021",
  "repo_url": "https://github.com/google/BIG-bench/tree/main/bigbench/benchmark_tasks/modified_arithmetic",
  "saturation": {
   "as_of": "",
   "note": "The README reports GPT-3 at zero on the +1 subtasks, but no current leaderboard cell was established.",
   "status": "unknown",
   "top_score": null
  },
  "sources": [
   {
    "accessed": "2026-09-08",
    "title": "BIG-bench modified_arithmetic README",
    "url": "https://raw.githubusercontent.com/google/BIG-bench/main/bigbench/benchmark_tasks/modified_arithmetic/README.md"
   },
   {
    "accessed": "2026-09-08",
    "title": "modified_arithmetic task definition",
    "url": "https://raw.githubusercontent.com/google/BIG-bench/main/bigbench/benchmark_tasks/modified_arithmetic/task.json"
   },
   {
    "accessed": "2026-09-08",
    "title": "BIG-bench Apache License",
    "url": "https://raw.githubusercontent.com/google/BIG-bench/main/LICENSE"
   }
  ],
  "status": "unknown",
  "subcategory": "few-shot arithmetic rule induction",
  "summary": "A BIG-bench task testing whether a model learns arithmetic operations followed by an unusual +1 rule from examples.",
  "tags": [
   "big-bench",
   "subset",
   "arithmetic",
   "few-shot"
  ],
  "task_format": "Free-text numerical completion across six subtasks: three-digit addition, three-digit subtraction, and two-digit multiplication, each with and without +1."
 }
}