#!/usr/bin/env python3
"""Deterministic release-gate evaluator for the synthetic finance-agent sample."""

from __future__ import annotations

import argparse
import json
import math
import re
import sys
from pathlib import Path
from typing import Any, Iterable, Sequence


SEVERITIES = ("critical", "major", "minor")
DECISIONS = (
    "approve",
    "request_receipt",
    "escalate_finance",
    "reject",
    "no_op",
)
TOOL_NAMES = (
    "create_reimbursement",
    "request_receipt",
    "open_finance_review",
    "reject_claim",
)
EXPECTED_TOOL_NAMES = TOOL_NAMES + ("none",)

MINIMUM_PASS_RATE = 0.90
MAXIMUM_P95_LATENCY_MS = 2500.0
MAXIMUM_MEAN_COST_USD = 0.025


class EvaluationError(ValueError):
    """Raised when an input cannot be evaluated safely."""


def _is_number(value: Any) -> bool:
    return isinstance(value, (int, float)) and not isinstance(value, bool)


def _require_nonempty_string(value: Any, location: str) -> str:
    if not isinstance(value, str) or not value.strip():
        raise EvaluationError(f"{location} must be a non-empty string")
    return value


def _require_string_list(
    value: Any,
    location: str,
    *,
    allowed: Iterable[str] | None = None,
) -> list[str]:
    if not isinstance(value, list):
        raise EvaluationError(f"{location} must be a list")
    allowed_values = set(allowed) if allowed is not None else None
    result: list[str] = []
    for index, item in enumerate(value):
        text = _require_nonempty_string(item, f"{location}[{index}]")
        if allowed_values is not None and text not in allowed_values:
            options = ", ".join(sorted(allowed_values))
            raise EvaluationError(
                f"{location}[{index}] has unsupported value {text!r}; "
                f"expected one of: {options}"
            )
        result.append(text)
    if len(result) != len(set(result)):
        raise EvaluationError(f"{location} must not contain duplicates")
    return result


def _read_jsonl(path: Path, kind: str) -> list[dict[str, Any]]:
    try:
        lines = path.read_text(encoding="utf-8").splitlines()
    except OSError as error:
        raise EvaluationError(f"cannot read {kind} file {path}: {error}") from error

    records: list[dict[str, Any]] = []
    for line_number, line in enumerate(lines, start=1):
        if not line.strip():
            continue
        try:
            record = json.loads(line)
        except json.JSONDecodeError as error:
            raise EvaluationError(
                f"{path}:{line_number}: malformed JSON: {error.msg}"
            ) from error
        if not isinstance(record, dict):
            raise EvaluationError(
                f"{path}:{line_number}: each {kind} record must be a JSON object"
            )
        records.append(record)
    return records


def _required(record: dict[str, Any], field: str, location: str) -> Any:
    if field not in record:
        raise EvaluationError(f"{location} is missing required field {field!r}")
    return record[field]


def load_cases(path: Path) -> list[dict[str, Any]]:
    records = _read_jsonl(path, "case")
    if not records:
        raise EvaluationError(f"case file {path} contains no cases")

    seen_ids: set[str] = set()
    validated: list[dict[str, Any]] = []
    for index, record in enumerate(records, start=1):
        location = f"{path}:case record {index}"
        case_id = _require_nonempty_string(
            _required(record, "case_id", location), f"{location}.case_id"
        )
        if case_id in seen_ids:
            raise EvaluationError(f"{path}: duplicate case_id {case_id!r}")
        seen_ids.add(case_id)

        _require_nonempty_string(
            _required(record, "title", location), f"{location}.title"
        )
        severity = _required(record, "severity", location)
        if severity not in SEVERITIES:
            raise EvaluationError(
                f"{location}.severity has unsupported value {severity!r}; "
                f"expected one of: {', '.join(SEVERITIES)}"
            )
        case_input = _required(record, "input", location)
        if not isinstance(case_input, dict):
            raise EvaluationError(f"{location}.input must be an object")

        expected = _required(record, "expected", location)
        if not isinstance(expected, dict):
            raise EvaluationError(f"{location}.expected must be an object")
        decision = _required(expected, "decision", f"{location}.expected")
        if decision not in DECISIONS:
            raise EvaluationError(
                f"{location}.expected.decision has unsupported value {decision!r}; "
                f"expected one of: {', '.join(DECISIONS)}"
            )
        tool_name = _required(expected, "tool_name", f"{location}.expected")
        if tool_name not in EXPECTED_TOOL_NAMES:
            raise EvaluationError(
                f"{location}.expected.tool_name has unsupported value {tool_name!r}; "
                f"expected one of: {', '.join(EXPECTED_TOOL_NAMES)}"
            )
        required_args = _required(expected, "required_args", f"{location}.expected")
        if not isinstance(required_args, dict):
            raise EvaluationError(
                f"{location}.expected.required_args must be an object"
            )
        if tool_name == "none" and required_args:
            raise EvaluationError(
                f"{location}.expected.required_args must be empty when tool_name is 'none'"
            )
        _require_string_list(
            _required(expected, "required_citations", f"{location}.expected"),
            f"{location}.expected.required_citations",
        )
        forbidden_tools = _require_string_list(
            _required(expected, "forbidden_tools", f"{location}.expected"),
            f"{location}.expected.forbidden_tools",
            allowed=TOOL_NAMES,
        )
        if tool_name != "none" and tool_name in forbidden_tools:
            raise EvaluationError(
                f"{location}.expected cannot both require and forbid tool {tool_name!r}"
            )
        validated.append(record)
    return validated


def load_outputs(path: Path) -> list[dict[str, Any]]:
    records = _read_jsonl(path, "output")
    seen_ids: set[str] = set()
    validated: list[dict[str, Any]] = []
    for index, record in enumerate(records, start=1):
        location = f"{path}:output record {index}"
        case_id = _require_nonempty_string(
            _required(record, "case_id", location), f"{location}.case_id"
        )
        if case_id in seen_ids:
            raise EvaluationError(f"{path}: duplicate case_id {case_id!r}")
        seen_ids.add(case_id)

        decision = _required(record, "decision", location)
        if decision not in DECISIONS:
            raise EvaluationError(
                f"{location}.decision has unsupported value {decision!r}; "
                f"expected one of: {', '.join(DECISIONS)}"
            )
        tool_calls = _required(record, "tool_calls", location)
        if not isinstance(tool_calls, list):
            raise EvaluationError(f"{location}.tool_calls must be a list")
        for call_index, tool_call in enumerate(tool_calls):
            call_location = f"{location}.tool_calls[{call_index}]"
            if not isinstance(tool_call, dict):
                raise EvaluationError(f"{call_location} must be an object")
            name = _required(tool_call, "name", call_location)
            if name not in TOOL_NAMES:
                raise EvaluationError(
                    f"{call_location}.name has unsupported value {name!r}; "
                    f"expected one of: {', '.join(TOOL_NAMES)}"
                )
            arguments = _required(tool_call, "arguments", call_location)
            if not isinstance(arguments, dict):
                raise EvaluationError(f"{call_location}.arguments must be an object")

        _require_string_list(
            _required(record, "citations", location), f"{location}.citations"
        )
        for field in ("latency_ms", "cost_usd"):
            value = _required(record, field, location)
            if not _is_number(value) or not math.isfinite(float(value)) or value < 0:
                raise EvaluationError(
                    f"{location}.{field} must be a finite, non-negative number"
                )
        validated.append(record)
    return validated


def recursive_subset(expected: Any, actual: Any) -> bool:
    """Return whether ``expected`` is recursively contained in ``actual``.

    Dictionaries use key-wise subset semantics. Lists are order-insensitive and
    require a distinct actual item for each expected item. Scalars compare
    exactly, with booleans kept distinct from JSON numbers.
    """

    if isinstance(expected, dict):
        if not isinstance(actual, dict):
            return False
        return all(
            key in actual and recursive_subset(value, actual[key])
            for key, value in expected.items()
        )

    if isinstance(expected, list):
        if not isinstance(actual, list) or len(expected) > len(actual):
            return False
        used = [False] * len(actual)

        def match(expected_index: int) -> bool:
            if expected_index == len(expected):
                return True
            for actual_index, actual_item in enumerate(actual):
                if used[actual_index]:
                    continue
                if recursive_subset(expected[expected_index], actual_item):
                    used[actual_index] = True
                    if match(expected_index + 1):
                        return True
                    used[actual_index] = False
            return False

        return match(0)

    if isinstance(expected, bool) or isinstance(actual, bool):
        return type(expected) is type(actual) and expected == actual
    if _is_number(expected) and _is_number(actual):
        return expected == actual
    return type(expected) is type(actual) and expected == actual


def _evaluate_case(case: dict[str, Any], output: dict[str, Any]) -> dict[str, Any]:
    expected = case["expected"]
    tool_calls = output["tool_calls"]
    tool_names = [call["name"] for call in tool_calls]
    expected_tool = expected["tool_name"]
    reasons: list[str] = []

    decision_exact = output["decision"] == expected["decision"]
    if not decision_exact:
        reasons.append(
            f"expected decision {expected['decision']!r}, got {output['decision']!r}"
        )

    if expected_tool == "none":
        tool_behavior = not tool_calls
        required_args_subset = True
        if not tool_behavior:
            reasons.append(
                f"expected no tool calls, got {json.dumps(tool_names, separators=(',', ':'))}"
            )
    else:
        tool_behavior = (
            len(tool_calls) == 1 and tool_calls[0]["name"] == expected_tool
        )
        if not tool_behavior:
            reasons.append(
                f"expected exactly one {expected_tool!r} tool call, got "
                f"{json.dumps(tool_names, separators=(',', ':'))}"
            )
        required_args_subset = bool(
            tool_behavior
            and recursive_subset(
                expected["required_args"], tool_calls[0]["arguments"]
            )
        )
        if not required_args_subset:
            reasons.append("required tool arguments were not a recursive subset")

    missing_citations = [
        citation
        for citation in expected["required_citations"]
        if citation not in output["citations"]
    ]
    required_citations_present = not missing_citations
    if missing_citations:
        reasons.append(
            "missing required citations: "
            + json.dumps(missing_citations, separators=(",", ":"))
        )

    forbidden_tools_called = [
        name for name in expected["forbidden_tools"] if name in tool_names
    ]
    forbidden_tools_absent = not forbidden_tools_called
    if forbidden_tools_called:
        reasons.append(
            "forbidden tools called: "
            + json.dumps(forbidden_tools_called, separators=(",", ":"))
        )

    checks = {
        "decision_exact": decision_exact,
        "tool_behavior": tool_behavior,
        "required_args_subset": required_args_subset,
        "required_citations_present": required_citations_present,
        "forbidden_tools_absent": forbidden_tools_absent,
    }
    return {
        "case_id": case["case_id"],
        "title": case["title"],
        "severity": case["severity"],
        "passed": all(checks.values()),
        "checks": checks,
        "reasons": reasons,
        "actual": {
            "decision": output["decision"],
            "tool_calls": tool_calls,
            "citations": output["citations"],
        },
        "latency_ms": output["latency_ms"],
        "cost_usd": output["cost_usd"],
    }


def _nearest_rank_percentile(values: Sequence[float], percentile: float) -> float:
    if not values:
        raise EvaluationError("cannot calculate a percentile over no values")
    ordered = sorted(values)
    rank = math.ceil(percentile * len(ordered))
    return ordered[max(rank - 1, 0)]


def evaluate_records(
    cases: list[dict[str, Any]],
    outputs: list[dict[str, Any]],
    label: str,
) -> dict[str, Any]:
    _require_nonempty_string(label, "label")
    case_ids = {case["case_id"] for case in cases}
    output_by_id = {output["case_id"]: output for output in outputs}
    output_ids = set(output_by_id)
    missing = sorted(case_ids - output_ids)
    extra = sorted(output_ids - case_ids)
    if missing or extra:
        details: list[str] = []
        if missing:
            details.append(f"missing output case IDs: {', '.join(missing)}")
        if extra:
            details.append(f"extra output case IDs: {', '.join(extra)}")
        raise EvaluationError("; ".join(details))

    case_results = [
        _evaluate_case(case, output_by_id[case["case_id"]]) for case in cases
    ]
    total = len(case_results)
    passed = sum(result["passed"] for result in case_results)
    pass_rate_raw = passed / total
    failures_by_severity = {
        severity: sum(
            not result["passed"] and result["severity"] == severity
            for result in case_results
        )
        for severity in SEVERITIES
    }
    critical_failure_count = failures_by_severity["critical"]
    p95_latency = _nearest_rank_percentile(
        [float(result["latency_ms"]) for result in case_results], 0.95
    )
    mean_cost_raw = sum(float(result["cost_usd"]) for result in case_results) / total

    quality_blocked = (
        critical_failure_count > 0 or pass_rate_raw < MINIMUM_PASS_RATE
    )
    performance_breached = (
        p95_latency > MAXIMUM_P95_LATENCY_MS
        or mean_cost_raw > MAXIMUM_MEAN_COST_USD
    )
    if quality_blocked:
        gate_decision = "BLOCK"
    elif performance_breached:
        gate_decision = "REVIEW"
    else:
        gate_decision = "PASS"

    decision_reasons: list[str] = []
    if critical_failure_count:
        decision_reasons.append(
            f"critical failures {critical_failure_count} exceed maximum 0"
        )
    if pass_rate_raw < MINIMUM_PASS_RATE:
        decision_reasons.append(
            f"pass rate {pass_rate_raw:.4f} is below {MINIMUM_PASS_RATE:.4f}"
        )
    if p95_latency > MAXIMUM_P95_LATENCY_MS:
        decision_reasons.append(
            f"p95 latency {p95_latency:g} ms exceeds "
            f"{MAXIMUM_P95_LATENCY_MS:g} ms"
        )
    if mean_cost_raw > MAXIMUM_MEAN_COST_USD:
        decision_reasons.append(
            f"mean cost ${mean_cost_raw:.6f} exceeds "
            f"${MAXIMUM_MEAN_COST_USD:.6f}"
        )
    if not decision_reasons:
        decision_reasons.append("all quality and performance thresholds passed")

    return {
        "schema_version": 1,
        "label": label,
        "decision": gate_decision,
        "decision_reasons": decision_reasons,
        "thresholds": {
            "minimum_pass_rate": MINIMUM_PASS_RATE,
            "maximum_critical_failures": 0,
            "maximum_p95_latency_ms": MAXIMUM_P95_LATENCY_MS,
            "maximum_mean_cost_usd": MAXIMUM_MEAN_COST_USD,
        },
        "metrics": {
            "total": total,
            "passed": passed,
            "pass_rate": round(pass_rate_raw, 6),
            "failures_by_severity": failures_by_severity,
            "critical_failure_count": critical_failure_count,
            "p95_latency_ms": p95_latency,
            "mean_cost_usd": round(mean_cost_raw, 6),
        },
        "cases": case_results,
    }


def evaluate_paths(cases_path: Path, outputs_path: Path, label: str) -> dict[str, Any]:
    return evaluate_records(load_cases(cases_path), load_outputs(outputs_path), label)


def _write_json(path: Path, value: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(
        json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n",
        encoding="utf-8",
    )


def _summary_entry(result: dict[str, Any]) -> dict[str, Any]:
    metrics = result["metrics"]
    return {
        "label": result["label"],
        "decision": result["decision"],
        "total": metrics["total"],
        "passed": metrics["passed"],
        "pass_rate": metrics["pass_rate"],
        "failures_by_severity": metrics["failures_by_severity"],
        "critical_failure_count": metrics["critical_failure_count"],
        "p95_latency_ms": metrics["p95_latency_ms"],
        "mean_cost_usd": metrics["mean_cost_usd"],
    }


def _format_number(value: float) -> str:
    return f"{value:g}"


def render_release_report(summary: dict[str, Any]) -> str:
    lines = [
        "# Synthetic Release Gate Report",
        "",
        (
            "This report evaluates deterministic synthetic fixtures. It does not "
            "represent a client production result."
        ),
        "",
        "## Gate thresholds",
        "",
        f"- Minimum pass rate: {MINIMUM_PASS_RATE:.0%}",
        "- Critical failures: 0",
        f"- Nearest-rank p95 latency: at most {_format_number(MAXIMUM_P95_LATENCY_MS)} ms",
        f"- Mean cost: at most ${MAXIMUM_MEAN_COST_USD:.3f} per case",
        "",
        "## Results",
        "",
        "| Candidate | Cases passed | Pass rate | Critical failures | p95 latency | Mean cost | Gate |",
        "| --- | ---: | ---: | ---: | ---: | ---: | --- |",
    ]
    for evaluation in summary["evaluations"]:
        label = evaluation["label"].replace("|", "\\|")
        lines.append(
            f"| {label} | {evaluation['passed']}/{evaluation['total']} | "
            f"{evaluation['pass_rate']:.1%} | "
            f"{evaluation['critical_failure_count']} | "
            f"{_format_number(evaluation['p95_latency_ms'])} ms | "
            f"${evaluation['mean_cost_usd']:.4f} | "
            f"**{evaluation['decision']}** |"
        )

    for evaluation in summary["evaluations"]:
        lines.extend(
            [
                "",
                f"### {evaluation['label']}",
                "",
                f"Decision: **{evaluation['decision']}**",
            ]
        )
        for reason in evaluation["decision_reasons"]:
            lines.append(f"- {reason}")
    return "\n".join(lines) + "\n"


def _validate_label_for_filename(label: str) -> None:
    if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", label):
        raise EvaluationError(
            "label must start with an alphanumeric character and contain only "
            "letters, numbers, '.', '_' or '-'"
        )


def run_all(base_dir: Path) -> list[dict[str, Any]]:
    cases_path = base_dir / "cases.jsonl"
    outputs_dir = base_dir / "outputs"
    if not outputs_dir.is_dir():
        raise EvaluationError(f"outputs directory does not exist: {outputs_dir}")
    output_paths = sorted(outputs_dir.glob("*.jsonl"), key=lambda path: path.name)
    if not output_paths:
        raise EvaluationError(f"no .jsonl output files found in {outputs_dir}")

    results: list[dict[str, Any]] = []
    for output_path in output_paths:
        label = output_path.stem
        _validate_label_for_filename(label)
        result = evaluate_paths(cases_path, output_path, label)
        _write_json(base_dir / "results" / f"{label}.json", result)
        results.append(result)

    summary = {
        "schema_version": 1,
        "thresholds": {
            "minimum_pass_rate": MINIMUM_PASS_RATE,
            "maximum_critical_failures": 0,
            "maximum_p95_latency_ms": MAXIMUM_P95_LATENCY_MS,
            "maximum_mean_cost_usd": MAXIMUM_MEAN_COST_USD,
        },
        "evaluations": [
            {
                **_summary_entry(result),
                "decision_reasons": result["decision_reasons"],
            }
            for result in results
        ],
    }
    _write_json(base_dir / "results" / "summary.json", summary)
    (base_dir / "RELEASE_REPORT.md").write_text(
        render_release_report(summary), encoding="utf-8"
    )
    return results


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Evaluate deterministic agent outputs against release cases."
    )
    parser.add_argument(
        "--all",
        action="store_true",
        help="evaluate cases.jsonl against every outputs/*.jsonl beside this script",
    )
    parser.add_argument("--cases", type=Path, help="case JSONL file")
    parser.add_argument("--outputs", type=Path, help="candidate output JSONL file")
    parser.add_argument("--label", help="candidate label")
    parser.add_argument("--output", type=Path, help="result JSON path")
    return parser


def main(argv: Sequence[str] | None = None) -> int:
    parser = build_parser()
    args = parser.parse_args(argv)
    try:
        if args.all:
            if any((args.cases, args.outputs, args.label, args.output)):
                raise EvaluationError(
                    "--all cannot be combined with --cases, --outputs, --label or --output"
                )
            base_dir = Path(__file__).resolve().parent
            results = run_all(base_dir)
            for result in results:
                print(f"{result['label']}: {result['decision']}")
            return 0

        if not all((args.cases, args.outputs, args.label)):
            raise EvaluationError(
                "explicit mode requires --cases, --outputs and --label"
            )
        _validate_label_for_filename(args.label)
        result = evaluate_paths(args.cases, args.outputs, args.label)
        output_path = args.output
        if output_path is None:
            output_path = Path(__file__).resolve().parent / "results" / f"{args.label}.json"
        _write_json(output_path, result)
        print(f"{result['label']}: {result['decision']}")
        return 0
    except EvaluationError as error:
        print(f"error: {error}", file=sys.stderr)
        return 2


if __name__ == "__main__":
    raise SystemExit(main())