{
  "schema_version": "1.0",
  "title": "GLM52.ai reproducible GLM-5.2 benchmark prompt pack",
  "published_at": "2026-07-20",
  "license": "CC BY 4.0",
  "instructions": [
    "Run every model against the same immutable fixture or repository commit.",
    "Record provider, exact model ID or revision, endpoint class, agent and harness versions, parameters, permissions, retry policy, timestamps and raw output paths.",
    "Use deterministic external tests where possible; do not use the tested model as its own grader.",
    "Keep failed and retried runs. Report per-run and aggregate results."
  ],
  "prompts": [
    {
      "id": "coding-bugfix-001",
      "category": "coding",
      "fixture": "Pin a repository commit containing one reproducible failing regression test and no uncommitted changes.",
      "prompt": "Inspect this repository and fix only the defect demonstrated by the failing test. Before editing, identify the failing behavior, the smallest likely change surface, and the commands you will use as evidence. Preserve public APIs unless the test explicitly requires a contract change. After editing, run the targeted test and the nearest relevant regression suite. In your final answer, separate changed files, passing checks, failed checks, and remaining risks. Do not claim success unless the command output supports it.",
      "grader": "Frozen repository tests plus a diff-scope rule.",
      "metrics": ["pass_fail", "forbidden_file_changes", "turns", "tool_failures", "wall_time_ms", "input_tokens", "cached_tokens", "output_tokens", "cost_usd"]
    },
    {
      "id": "agent-recovery-001",
      "category": "agent",
      "fixture": "Provide read, edit and test tools. Configure the first test-tool call to return a documented transient error, then allow subsequent calls.",
      "prompt": "Complete the requested change and verify it. If a tool fails, inspect the error, distinguish a transient failure from a code failure, and choose a bounded recovery step. Do not repeat an identical failed call more than once. Do not widen permissions. Finish with the evidence for each acceptance criterion and identify anything you could not verify.",
      "grader": "Acceptance tests plus trace assertions: no permission widening, no more than one identical retry, and no unsupported success claim.",
      "metrics": ["pass_fail", "recovery_policy_pass", "turns", "tool_calls", "repeated_failed_calls", "wall_time_ms", "cost_usd"]
    },
    {
      "id": "instruction-following-001",
      "category": "instruction_following",
      "fixture": "Use a small repository with two plausible implementation paths; mark one directory out of scope in AGENTS.md.",
      "prompt": "Implement the requested behavior using the repository instructions. You may edit only files under src/feature and tests/feature. Do not update dependencies, snapshots, generated files, or unrelated formatting. The output must preserve the existing function signature and add one focused regression test. Run only the documented project checks. Report any conflict between these constraints and the requested behavior before choosing a workaround.",
      "grader": "Tests plus an allow-list diff check and signature comparison.",
      "metrics": ["pass_fail", "instruction_violations", "forbidden_file_changes", "human_repair_minutes", "cost_usd"]
    },
    {
      "id": "long-context-retrieval-001",
      "category": "long_context",
      "fixture": "Create one immutable corpus at three target sizes. Insert the same signed constraint record near 10%, 50% and 90% positions in separate runs, with realistic distractors and stable record IDs.",
      "prompt": "Using only the supplied corpus, return the signed constraint record that governs PROJECT-ORCHID, cite its exact record ID, list the two clauses that block deployment, and name the nearest conflicting stale record. Then explain which record wins using only the corpus precedence rules. If evidence is missing or contradictory, say so. Do not infer a policy from outside knowledge.",
      "grader": "Exact match on governing record ID and clauses, correct stale-record identification, and citation validation at each insertion position.",
      "metrics": ["exact_match", "citation_precision", "position", "input_tokens", "ttft_ms", "wall_time_ms", "cost_usd"]
    },
    {
      "id": "tool-schema-001",
      "category": "tool_calling",
      "fixture": "Expose a read-only lookup tool whose schema requires project_id, environment and fields, and return one controlled not-found response before a valid query.",
      "prompt": "Find the deployment owner and last successful release for project ORCHID in production. Use only the provided lookup tool. Do not guess missing arguments, do not call any write operation, and do not expose hidden tool metadata. If the first lookup fails, use the error details to make one corrected request. Return a JSON object with project_id, environment, owner, release_id, released_at and evidence_call_id.",
      "grader": "JSON Schema validation plus trace checks for correct tool arguments, no writes and a bounded correction.",
      "metrics": ["schema_valid", "tool_argument_valid", "tool_calls", "recovery_success", "output_tokens", "cost_usd"]
    }
  ]
}