#!/usr/bin/env python3
# Published reference script.
"""Run the hosted model over 20 questions, Q01 through Q20.

Examples:
    export OPENROUTER_API_KEY=...
    python3 runfrontier.py --only q01
    python3 runfrontier.py

Historical Series 1 disclosure: Q12 received a second hosted attempt after the first attempt was blocked. The completed rerun was scored. The first blocked artifact and full routing record are not distributed.

The request sends no sampling parameters, so gateway or provider defaults govern.
A content_filter or other non-stop finish reason is preserved and returned without
an automatic do-over. It is never silently repaired or replaced. Reruns require a
documented rule or an explicit disclosure.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import pathlib
import sys
import time
from typing import Any

import requests


def classify_finish(finish_reason: str | None, transport_failed: bool, refusal: bool = False) -> dict[str, Any]:
    return {
        "transport_failed": transport_failed,
        "finish_reason": finish_reason,
        "length_truncated": finish_reason == "length",
        "delivery_complete": (not transport_failed) and finish_reason == "stop" and not refusal,
        "delivery_failure": transport_failed or refusal or finish_reason != "stop",
        "refusal_detected": refusal,
        "scored_as_delivered": True,
    }


def ask(url: str, key: str, model: str, prompt: str, max_tokens: int, timeout: int) -> tuple[dict[str, Any], list[dict[str, Any]]]:
    headers = {"Authorization": "Bearer " + key, "Content-Type": "application/json"}
    payload = {"model": model, "messages": [{"role": "user", "content": prompt}], "max_tokens": max_tokens, "stream": False}
    history: list[dict[str, Any]] = []
    for attempt in range(1, 4):
        try:
            response = requests.post(url, headers=headers, json=payload, timeout=timeout)
            if response.status_code in (429, 500, 502, 503, 504):
                raise requests.HTTPError(f"HTTP {response.status_code}: {response.text[:300]}")
            response.raise_for_status()
            data = response.json()
            if "error" in data:
                raise RuntimeError(str(data["error"])[:300])
            choice = data["choices"][0]
            message = choice.get("message") or {}
            finish = choice.get("finish_reason")
            refusal = bool(message.get("refusal"))
            history.append({"attempt": attempt, "transport_failed": False, "finish_reason": finish, "superseded": False})
            result = {
                "content": message.get("content") or "",
                "reasoning": message.get("reasoning") or message.get("reasoning_content") or "",
                "server_model": data.get("model"),
                "provider": data.get("provider"),
                "usage": data.get("usage") or {},
                "attempt_number": attempt,
                "rerun_occurred": attempt > 1,
                "rerun_reason": "transport retry" if attempt > 1 else None,
                "scored_attempt": attempt,
                "earlier_attempt_superseded": attempt > 1,
            }
            result.update(classify_finish(finish, False, refusal))
            return result, history
        except Exception as exc:
            history.append({"attempt": attempt, "transport_failed": True, "error": str(exc), "superseded": attempt < 3})
            if attempt == 3:
                raise
            time.sleep(10 * attempt)
    raise RuntimeError("unreachable")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", default="anthropic/claude-opus-5")
    parser.add_argument("--label", default="claude-opus-5")
    parser.add_argument("--base-url", default="https://openrouter.ai/api/v1")
    parser.add_argument("--questions", default="questions/")
    parser.add_argument("--answers", default="answers/")
    parser.add_argument("--logs", default="logs/")
    parser.add_argument("--only")
    parser.add_argument("--max-tokens", type=int, default=64000)
    parser.add_argument("--timeout", type=int, default=3600)
    args = parser.parse_args()

    key = os.environ.get("OPENROUTER_API_KEY", "").strip()
    if not key:
        raise SystemExit("OPENROUTER_API_KEY is not set")
    base = pathlib.Path(__file__).resolve().parent
    questions = sorted((base / args.questions).glob("q*.md"))
    if args.only:
        questions = [p for p in questions if p.stem == args.only]
    if not questions:
        raise SystemExit("No matching question files")

    answer_dir = base / args.answers / args.label
    answer_dir.mkdir(parents=True, exist_ok=True)
    log_dir = base / args.logs
    log_dir.mkdir(parents=True, exist_ok=True)
    log_file = log_dir / "runs.jsonl"

    for question_path in questions:
        answer_path = answer_dir / f"{args.label}__{question_path.stem}_r1.md"
        if answer_path.exists():
            continue
        result, attempts = ask(args.base_url.rstrip("/") + "/chat/completions", key, args.model, question_path.read_text(encoding="utf-8"), args.max_tokens, args.timeout)
        answer_path.write_text(result.pop("content"), encoding="utf-8")
        reasoning = result.pop("reasoning")
        if reasoning:
            answer_path.with_suffix(".reasoning.md").write_text(reasoning, encoding="utf-8")
        record = {
            **result,
            "attempt_history": attempts,
            "label": args.label,
            "question": question_path.stem,
            "rep": 1,
            "max_tokens": args.max_tokens,
            "base_url": args.base_url,
            "answer_file": str(answer_path.relative_to(base)),
            "answer_sha256": hashlib.sha256(answer_path.read_bytes()).hexdigest(),
            "sampling_parameters_sent": None,
            "_public_record": {"type": "prospective reproduction record", "policy": "../../PUBLIC_RECORD_POLICY.md"},
        }
        with log_file.open("a", encoding="utf-8") as f:
            f.write(json.dumps(record, ensure_ascii=False) + "\n")
        print(question_path.stem, args.label, record["finish_reason"], "delivery_complete=", record["delivery_complete"])


if __name__ == "__main__":
    main()