{
  "scope": {
    "units_per_model": 20,
    "unique_case_ids": 16,
    "distinct_repeated_case_ids": [
      "plan-clinic-privacy",
      "plan-clinic-urgent",
      "plan-rent-and-work",
      "plan-rent-only"
    ],
    "models": [
      "beam",
      "deepseek",
      "glm",
      "qwen38",
      "kimi",
      "minimax",
      "gemma",
      "gptoss"
    ],
    "case_units": [
      "base7:plan-fees-no-borrowing",
      "base7:plan-rent-and-work",
      "base7:plan-rent-only",
      "base7:plan-clinic-urgent",
      "base7:plan-clinic-privacy",
      "base7:plan-no-eligible-option",
      "base7:plan-untrusted-document",
      "add13:plan-carer-service",
      "add13:plan-carer-money",
      "add13:plan-training-not-student",
      "add13:plan-training-acceptance-unknown",
      "add13:plan-debt-no-new-loan",
      "add13:plan-debt-consult-only",
      "add13:plan-emergency-today",
      "add13:plan-emergency-recovery",
      "add13:plan-fees-alternatives",
      "add13:plan-rent-and-work",
      "add13:plan-rent-only",
      "add13:plan-clinic-urgent",
      "add13:plan-clinic-privacy"
    ],
    "model_configurations": [
      {
        "id": "beam",
        "provider": "beam",
        "model": "reflection/Beam-1.0-501B-A23B:baseten",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.4,
          "outputUsdPerMillion": 1.2,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.442Z"
        }
      },
      {
        "id": "deepseek",
        "provider": "beam",
        "model": "deepseek-ai/DeepSeek-V4.1-Flash:novita",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.3,
          "outputUsdPerMillion": 1.2,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.442Z"
        }
      },
      {
        "id": "glm",
        "provider": "beam",
        "model": "zai-org/GLM-5.3-Flash:novita",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.15,
          "outputUsdPerMillion": 0.5,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.442Z"
        }
      },
      {
        "id": "qwen38",
        "provider": "beam",
        "model": "Qwen/Qwen3.8-Flash-Next-FP8:nscale",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.15,
          "outputUsdPerMillion": 0.47,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.442Z"
        }
      },
      {
        "id": "kimi",
        "provider": "beam",
        "model": "moonshotai/Kimi-K2.5:deepinfra",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.45,
          "outputUsdPerMillion": 2.25,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.442Z"
        }
      },
      {
        "id": "minimax",
        "provider": "beam",
        "model": "MiniMaxAI/MiniMax-M3:deepinfra",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.28,
          "outputUsdPerMillion": 1.1,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.442Z"
        }
      },
      {
        "id": "gemma",
        "provider": "beam",
        "model": "google/gemma-4-31B-it:novita",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.14,
          "outputUsdPerMillion": 0.4,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.443Z"
        }
      },
      {
        "id": "gptoss",
        "provider": "beam",
        "model": "openai/gpt-oss-120b:novita",
        "enabled": true,
        "inputTokenCeiling": 32768,
        "maxOutputTokens": 4096,
        "price": {
          "inputUsdPerMillion": 0.05,
          "outputUsdPerMillion": 0.25,
          "source": "https://huggingface.co/inference/models",
          "verifiedAt": "2026-10-31T18:55:35.443Z"
        }
      }
    ]
  },
  "rules": {
    "score": "existing independent LLM judgment usefulness 0-10; completed scored only; no failure imputation",
    "completion": "selected outcome pending_human_review AND finishReason stop AND parsed answer present; denominator all 20",
    "first_attempt": "original pre-retry result, regardless of eventual success",
    "duration": "selected result elapsedMs /1000; mean completed only; retries not added",
    "cost": "logged estimatedCostUsd for every generation attempt; null if any attempt cost unknown; known subtotal separately; excludes judge costs; reasoning token separate charge unverified, do not add separately",
    "win": "sole top among all 8 scored completed models per unit / fully comparable units; ties separate",
    "judgment_join": "selected raw result line SHA256 == judgment.sourceDigest; duplicate valid judgments abort",
    "selection": "original 160 attempts plus only documented 3 GLM, 2 Kimi add13, 1 Kimi base7 retries; replace initial only if complete"
  },
  "models": {
    "beam": {
      "model": "Beam",
      "judge_model_counts": {
        "minimax": 17,
        "deepseek": 3
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 7.95,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "pending_human_review": 20
      },
      "first_attempt_complete_count": 20,
      "first_attempt_completion_rate": 1.0,
      "selected_mean_seconds": 2.9515000000000002,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.037391999999999995,
      "selected_generation_cost_mean_usd": 0.0018696,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": 0.037391999999999995,
      "all_attempts_cost_mean_per_unit_usd": 0.0018695999999999997,
      "known_attempt_cost_subtotal_usd": 0.037391999999999995,
      "unknown_cost_attempts": 0,
      "attempt_count": 20,
      "sole_first_count": 2,
      "tied_first_count": 5,
      "win_denominator": 20,
      "sole_win_rate": 0.1
    },
    "deepseek": {
      "model": "DeepSeek",
      "judge_model_counts": {
        "beam": 18,
        "minimax": 2
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 8.3,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "pending_human_review": 20
      },
      "first_attempt_complete_count": 20,
      "first_attempt_completion_rate": 1.0,
      "selected_mean_seconds": 13.16445,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.0426363,
      "selected_generation_cost_mean_usd": 0.002131815,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": 0.0426363,
      "all_attempts_cost_mean_per_unit_usd": 0.002131815,
      "known_attempt_cost_subtotal_usd": 0.0426363,
      "unknown_cost_attempts": 0,
      "attempt_count": 20,
      "sole_first_count": 3,
      "tied_first_count": 7,
      "win_denominator": 20,
      "sole_win_rate": 0.15
    },
    "glm": {
      "model": "GLM",
      "judge_model_counts": {
        "beam": 19,
        "minimax": 1
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 8.3,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "pending_human_review": 17,
        "truncated": 3
      },
      "first_attempt_complete_count": 17,
      "first_attempt_completion_rate": 0.85,
      "selected_mean_seconds": 40.92655,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.0306355,
      "selected_generation_cost_mean_usd": 0.001531775,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": 0.0371134,
      "all_attempts_cost_mean_per_unit_usd": 0.0018556699999999998,
      "known_attempt_cost_subtotal_usd": 0.0371134,
      "unknown_cost_attempts": 0,
      "attempt_count": 23,
      "sole_first_count": 2,
      "tied_first_count": 8,
      "win_denominator": 20,
      "sole_win_rate": 0.1
    },
    "qwen38": {
      "model": "Qwen",
      "judge_model_counts": {
        "minimax": 2,
        "beam": 18
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 7.95,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "pending_human_review": 20
      },
      "first_attempt_complete_count": 20,
      "first_attempt_completion_rate": 1.0,
      "selected_mean_seconds": 9.7234,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.01798702,
      "selected_generation_cost_mean_usd": 0.000899351,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": 0.01798702,
      "all_attempts_cost_mean_per_unit_usd": 0.000899351,
      "known_attempt_cost_subtotal_usd": 0.01798702,
      "unknown_cost_attempts": 0,
      "attempt_count": 20,
      "sole_first_count": 0,
      "tied_first_count": 4,
      "win_denominator": 20,
      "sole_win_rate": 0.0
    },
    "kimi": {
      "model": "Kimi",
      "judge_model_counts": {
        "beam": 18,
        "minimax": 2
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 7.9,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "transport_or_timeout": 1,
        "pending_human_review": 17,
        "http_error": 2
      },
      "first_attempt_complete_count": 17,
      "first_attempt_completion_rate": 0.85,
      "selected_mean_seconds": 91.7133,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.12164130000000001,
      "selected_generation_cost_mean_usd": 0.006082065,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": null,
      "all_attempts_cost_mean_per_unit_usd": null,
      "known_attempt_cost_subtotal_usd": 0.12164130000000001,
      "unknown_cost_attempts": 3,
      "attempt_count": 23,
      "sole_first_count": 1,
      "tied_first_count": 4,
      "win_denominator": 20,
      "sole_win_rate": 0.05
    },
    "minimax": {
      "model": "MiniMax",
      "judge_model_counts": {
        "beam": 20
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 7.95,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "pending_human_review": 20
      },
      "first_attempt_complete_count": 20,
      "first_attempt_completion_rate": 1.0,
      "selected_mean_seconds": 59.5184,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.0171102,
      "selected_generation_cost_mean_usd": 0.0008555100000000001,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": 0.0171102,
      "all_attempts_cost_mean_per_unit_usd": 0.00085551,
      "known_attempt_cost_subtotal_usd": 0.0171102,
      "unknown_cost_attempts": 0,
      "attempt_count": 20,
      "sole_first_count": 1,
      "tied_first_count": 5,
      "win_denominator": 20,
      "sole_win_rate": 0.05
    },
    "gemma": {
      "model": "Gemma",
      "judge_model_counts": {
        "minimax": 3,
        "beam": 17
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 7.75,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "pending_human_review": 20
      },
      "first_attempt_complete_count": 20,
      "first_attempt_completion_rate": 1.0,
      "selected_mean_seconds": 8.62445,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.0034097400000000005,
      "selected_generation_cost_mean_usd": 0.000170487,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": 0.0034097400000000005,
      "all_attempts_cost_mean_per_unit_usd": 0.00017048700000000004,
      "known_attempt_cost_subtotal_usd": 0.0034097400000000005,
      "unknown_cost_attempts": 0,
      "attempt_count": 20,
      "sole_first_count": 0,
      "tied_first_count": 5,
      "win_denominator": 20,
      "sole_win_rate": 0.0
    },
    "gptoss": {
      "model": "GPT-OSS",
      "judge_model_counts": {
        "beam": 17,
        "minimax": 3
      },
      "target_count": 20,
      "normal_complete_count": 20,
      "completion_rate": 1.0,
      "scored_count": 20,
      "average_score": 7.1,
      "failure_adjusted_score": null,
      "initial_status_counts": {
        "pending_human_review": 20
      },
      "first_attempt_complete_count": 20,
      "first_attempt_completion_rate": 1.0,
      "selected_mean_seconds": 12.4184,
      "time_count": 20,
      "selected_generation_cost_sum_usd": 0.006478050000000002,
      "selected_generation_cost_mean_usd": 0.0003239025,
      "selected_cost_count": 20,
      "all_attempts_cost_sum_usd": 0.006478050000000002,
      "all_attempts_cost_mean_per_unit_usd": 0.0003239025000000001,
      "known_attempt_cost_subtotal_usd": 0.006478050000000002,
      "unknown_cost_attempts": 0,
      "attempt_count": 20,
      "sole_first_count": 0,
      "tied_first_count": 4,
      "win_denominator": 20,
      "sole_win_rate": 0.0
    }
  },
  "disclosure": "This export omits local log filenames and raw model outputs. Scores are model-judged; source logs are not published. Prices are recorded estimates, not verified billing."
}