{
  "model": "subq-snap-preview",
  "created_at": "2026-10-13T02:15:32.780669+00:00",
  "live": true,
  "plan": {
    "seed": 20261013,
    "pairs": 24,
    "requests": 48,
    "reservation_usd": 0.9437184000000001,
    "budget_usd": 2.0,
    "context_tokens_reserved_per_request": 393216,
    "source_hashes": {
      "quasi-riemann": "42a5ee0febca59fd1def55cfd6c6808c322ef4237d7726303ea52f711deac6a3",
      "noncoordinate-polynomial": "60993c0b524291b7f707e42eec89ec7e910c22760d81f1d3af007c229228c13c",
      "foulkes-howe-stabilization": "89b46306d4bc756ff5fc261deaf3075ce1afa37087a313bab7abaf1569b8af21"
    }
  },
  "groups": [
    {
      "paper_slug": "foulkes-howe-stabilization",
      "context_mode": "excerpt",
      "scheduled": 8,
      "attempted": 8,
      "valid_answers": 8,
      "reference_agreement": 7,
      "median_latency_seconds": 0.1264,
      "reported_input_tokens": 3425,
      "max_reported_input_tokens": 533,
      "choices": {
        "background": 3,
        "proof_input": 5
      },
      "confusion": {
        "background -> background": 2,
        "proof_input -> proof_input": 5,
        "insufficient_context -> background": 1
      }
    },
    {
      "paper_slug": "foulkes-howe-stabilization",
      "context_mode": "full_manuscript",
      "scheduled": 8,
      "attempted": 8,
      "valid_answers": 8,
      "reference_agreement": 8,
      "median_latency_seconds": 0.27990000000000004,
      "reported_input_tokens": 70585,
      "max_reported_input_tokens": 8928,
      "choices": {
        "proof_input": 5,
        "background": 3
      },
      "confusion": {
        "proof_input -> proof_input": 5,
        "background -> background": 3
      }
    },
    {
      "paper_slug": "noncoordinate-polynomial",
      "context_mode": "excerpt",
      "scheduled": 8,
      "attempted": 8,
      "valid_answers": 8,
      "reference_agreement": 6,
      "median_latency_seconds": 0.13440000000000002,
      "reported_input_tokens": 3139,
      "max_reported_input_tokens": 493,
      "choices": {
        "background": 4,
        "proof_input": 4
      },
      "confusion": {
        "background -> background": 3,
        "insufficient_context -> background": 1,
        "proof_input -> proof_input": 3,
        "insufficient_context -> proof_input": 1
      }
    },
    {
      "paper_slug": "noncoordinate-polynomial",
      "context_mode": "full_manuscript",
      "scheduled": 8,
      "attempted": 8,
      "valid_answers": 8,
      "reference_agreement": 8,
      "median_latency_seconds": 0.275,
      "reported_input_tokens": 60979,
      "max_reported_input_tokens": 7723,
      "choices": {
        "background": 4,
        "proof_input": 4
      },
      "confusion": {
        "background -> background": 4,
        "proof_input -> proof_input": 4
      }
    },
    {
      "paper_slug": "quasi-riemann",
      "context_mode": "excerpt",
      "scheduled": 8,
      "attempted": 8,
      "valid_answers": 8,
      "reference_agreement": 8,
      "median_latency_seconds": 0.12825,
      "reported_input_tokens": 4005,
      "max_reported_input_tokens": 596,
      "choices": {
        "background": 3,
        "proof_input": 3,
        "explicitly_excluded": 2
      },
      "confusion": {
        "background -> background": 3,
        "proof_input -> proof_input": 3,
        "explicitly_excluded -> explicitly_excluded": 2
      }
    },
    {
      "paper_slug": "quasi-riemann",
      "context_mode": "full_manuscript",
      "scheduled": 8,
      "attempted": 8,
      "valid_answers": 8,
      "reference_agreement": 8,
      "median_latency_seconds": 2.2767999999999997,
      "reported_input_tokens": 2149257,
      "max_reported_input_tokens": 268753,
      "choices": {
        "background": 3,
        "explicitly_excluded": 2,
        "proof_input": 3
      },
      "confusion": {
        "background -> background": 3,
        "explicitly_excluded -> explicitly_excluded": 2,
        "proof_input -> proof_input": 3
      }
    }
  ],
  "paired": [
    {
      "pair_id": "quasi-riemann-source-1",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-5",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "quasi-riemann-source-0",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-4",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-6",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-7",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "insufficient_context",
      "full_expected_role": "background"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-3",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-5",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "quasi-riemann-source-4",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-6",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "quasi-riemann-source-5",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "explicitly_excluded",
      "full_choice": "explicitly_excluded",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "explicitly_excluded",
      "full_expected_role": "explicitly_excluded"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-1",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "quasi-riemann-source-6",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-7",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "insufficient_context",
      "full_expected_role": "background"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-2",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "quasi-riemann-source-2",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "explicitly_excluded",
      "full_choice": "explicitly_excluded",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "explicitly_excluded",
      "full_expected_role": "explicitly_excluded"
    },
    {
      "pair_id": "quasi-riemann-source-3",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-3",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "insufficient_context",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-2",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-8",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "noncoordinate-polynomial-role-8",
      "paper_slug": "noncoordinate-polynomial",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-1",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    },
    {
      "pair_id": "quasi-riemann-source-7",
      "paper_slug": "quasi-riemann",
      "excerpt_choice": "background",
      "full_choice": "background",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "background",
      "full_expected_role": "background"
    },
    {
      "pair_id": "foulkes-howe-stabilization-role-4",
      "paper_slug": "foulkes-howe-stabilization",
      "excerpt_choice": "proof_input",
      "full_choice": "proof_input",
      "changed": false,
      "excerpt_matches_full_reference": true,
      "full_matches_full_reference": true,
      "excerpt_expected_role": "proof_input",
      "full_expected_role": "proof_input"
    }
  ],
  "completed_pairs": 24,
  "changed_decisions": 0,
  "toward_full_reference": 0,
  "away_from_full_reference": 0,
  "known_usage_usd": 0.1145695,
  "reported_input_tokens": 2291390,
  "errors": 0,
  "limitations": "One request per condition, fixed option order, selected source-role cases with AI-assisted labels rather than independent human gold. Context-specific expected labels can differ. Long source contains manuscript only, never panel findings. This is dependency-role agreement, not mathematical correctness or a general long-context benchmark."
}