#!/usr/bin/env python3
# Published reference script.
"""Run the hosted model over 60 questions, Q21 through Q80.

Examples:
    export OPENROUTER_API_KEY=...
    python3 runfrontier.py --only q21
    python3 runfrontier.py



The request sends no sampling parameters, so gateway or provider defaults govern.
A content_filter or other non-stop finish reason is preserved and returned without
an automatic do-over. It is never silently repaired or replaced. Reruns require a
documented rule or an explicit disclosure.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import pathlib
import sys
import time
from typing import Any

import requests


def classify_finish(finish_reason: str | None, transport_failed: bool, refusal: bool = False) -> dict[str, Any]:
    return {
        "transport_failed": transport_failed,
        "finish_reason": finish_reason,
        "length_truncated": finish_reason == "length",
        "delivery_complete": (not transport_failed) and finish_reason == "stop" and not refusal,
        "delivery_failure": transport_failed or refusal or finish_reason != "stop",
        "refusal_detected": refusal,
        "scored_as_delivered": True,
    }


def ask(url: str, key: str, model: str, prompt: str, max_tokens: int, timeout: int) -> tuple[dict[str, Any], list[dict[str, Any]]]:
    headers = {"Authorization": "Bearer " + key, "Content-Type": "application/json"}
    payload = {"model": model, "messages": [{"role": "user", "content": prompt}], "max_tokens": max_tokens, "stream": False}
    history: list[dict[str, Any]] = []
    for attempt in range(1, 4):
        try:
            response = requests.post(url, headers=headers, json=payload, timeout=timeout)
            if response.status_code in (429, 500, 502, 503, 504):
                raise requests.HTTPError(f"HTTP {response.status_code}: {response.text[:300]}")
            response.raise_for_status()
            data = response.json()
            if "error" in data:
                raise RuntimeError(str(data["error"])[:300])
            choice = data["choices"][0]
            message = choice.get("message") or {}
            finish = choice.get("finish_reason")
            refusal = bool(message.get("refusal"))
            history.append({"attempt": attempt, "transport_failed": False, "finish_reason": finish, "superseded": False})
            result = {
                "content": message.get("content") or "",
                "reasoning": message.get("reasoning") or message.get("reasoning_content") or "",
                "server_model": data.get("model"),
                "provider": data.get("provider"),
                "usage": data.get("usage") or {},
                "attempt_number": attempt,
                "rerun_occurred": attempt > 1,
                "rerun_reason": "transport retry" if attempt > 1 else None,
                "scored_attempt": attempt,
                "earlier_attempt_superseded": attempt > 1,
            }
            result.update(classify_finish(finish, False, refusal))
            return result, history
        except Exception as exc:
            history.append({"attempt": attempt, "transport_failed": True, "error": str(exc), "superseded": attempt < 3})
            if attempt == 3:
                raise
            time.sleep(10 * attempt)
    raise RuntimeError("unreachable")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", default="anthropic/claude-opus-5")
    parser.add_argument("--label", default="claude-opus-5")
    parser.add_argument("--base-url", default="https://openrouter.ai/api/v1")
    parser.add_argument("--questions", default="questions/")
    parser.add_argument("--answers", default="answers/")
    parser.add_argument("--logs", default="logs/")
    parser.add_argument("--only")
    parser.add_argument("--max-tokens", type=int, default=64000)
    parser.add_argument("--timeout", type=int, default=3600)
    args = parser.parse_args()

    key = os.environ.get("OPENROUTER_API_KEY", "").strip()
    if not key:
        raise SystemExit("OPENROUTER_API_KEY is not set")
    base = pathlib.Path(__file__).resolve().parent
    questions = sorted((base / args.questions).glob("q*.md"))
    if args.only:
        questions = [p for p in questions if p.stem == args.only]
    if not questions:
        raise SystemExit("No matching question files")

    answer_dir = base / args.answers / args.label
    answer_dir.mkdir(parents=True, exist_ok=True)
    log_dir = base / args.logs
    log_dir.mkdir(parents=True, exist_ok=True)
    log_file = log_dir / "runs.jsonl"

    for question_path in questions:
        answer_path = answer_dir / f"{args.label}__{question_path.stem}_r1.md"
        if answer_path.exists():
            continue
        result, attempts = ask(args.base_url.rstrip("/") + "/chat/completions", key, args.model, question_path.read_text(encoding="utf-8"), args.max_tokens, args.timeout)
        answer_path.write_text(result.pop("content"), encoding="utf-8")
        reasoning = result.pop("reasoning")
        if reasoning:
            answer_path.with_suffix(".reasoning.md").write_text(reasoning, encoding="utf-8")
        record = {
            **result,
            "attempt_history": attempts,
            "label": args.label,
            "question": question_path.stem,
            "rep": 1,
            "max_tokens": args.max_tokens,
            "base_url": args.base_url,
            "answer_file": str(answer_path.relative_to(base)),
            "answer_sha256": hashlib.sha256(answer_path.read_bytes()).hexdigest(),
            "sampling_parameters_sent": None,
            "_public_record": {"type": "prospective reproduction record", "policy": "../../PUBLIC_RECORD_POLICY.md"},
        }
        with log_file.open("a", encoding="utf-8") as f:
            f.write(json.dumps(record, ensure_ascii=False) + "\n")
        print(question_path.stem, args.label, record["finish_reason"], "delivery_complete=", record["delivery_complete"])


if __name__ == "__main__":
    main()