from __future__ import annotations

import json
import sys
import tempfile
import unittest
from pathlib import Path


SAMPLE_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SAMPLE_DIR))

import evaluate  # noqa: E402


def make_case(
    case_id: str,
    *,
    severity: str = "minor",
    decision: str = "approve",
    tool_name: str = "none",
    required_args: dict | None = None,
    required_citations: list[str] | None = None,
    forbidden_tools: list[str] | None = None,
) -> dict:
    return {
        "case_id": case_id,
        "title": f"Case {case_id}",
        "severity": severity,
        "input": {"claim_id": case_id},
        "expected": {
            "decision": decision,
            "tool_name": tool_name,
            "required_args": required_args or {},
            "required_citations": required_citations or [],
            "forbidden_tools": forbidden_tools or [],
        },
    }


def make_output(
    case_id: str,
    *,
    decision: str = "approve",
    tool_calls: list[dict] | None = None,
    citations: list[str] | None = None,
    latency_ms: float = 1000,
    cost_usd: float = 0.01,
) -> dict:
    return {
        "case_id": case_id,
        "decision": decision,
        "tool_calls": tool_calls or [],
        "citations": citations or [],
        "latency_ms": latency_ms,
        "cost_usd": cost_usd,
    }


class RecursiveSubsetTests(unittest.TestCase):
    def test_nested_dict_and_order_independent_distinct_list_items(self) -> None:
        expected = {
            "claim": {
                "amount": 42,
                "lines": [{"code": "meal"}, {"code": "meal", "limit": 20}],
            }
        }
        actual = {
            "claim": {
                "amount": 42.0,
                "currency": "USD",
                "lines": [
                    {"code": "meal", "limit": 20, "receipt": True},
                    {"code": "meal", "limit": 10},
                ],
            },
            "audit": True,
        }
        self.assertTrue(evaluate.recursive_subset(expected, actual))

    def test_list_items_cannot_reuse_one_actual_item(self) -> None:
        self.assertFalse(evaluate.recursive_subset([{"x": 1}, {"x": 1}], [{"x": 1}]))

    def test_boolean_is_not_a_number(self) -> None:
        self.assertFalse(evaluate.recursive_subset(True, 1))


class GateDecisionTests(unittest.TestCase):
    def test_pass(self) -> None:
        cases = [make_case(f"c{index}") for index in range(1, 11)]
        outputs = [make_output(case["case_id"]) for case in cases]
        result = evaluate.evaluate_records(cases, outputs, "candidate_pass")
        self.assertEqual(result["decision"], "PASS")
        self.assertEqual(result["metrics"]["passed"], 10)
        self.assertEqual(result["metrics"]["p95_latency_ms"], 1000.0)

    def test_review_for_performance_only(self) -> None:
        cases = [make_case(f"c{index}") for index in range(1, 11)]
        outputs = [make_output(case["case_id"]) for case in cases]
        outputs[-1]["latency_ms"] = 2501
        result = evaluate.evaluate_records(cases, outputs, "candidate_review")
        self.assertEqual(result["decision"], "REVIEW")
        self.assertEqual(result["metrics"]["passed"], 10)
        self.assertEqual(result["metrics"]["p95_latency_ms"], 2501.0)

    def test_block_for_pass_rate(self) -> None:
        cases = [make_case(f"c{index}", severity="major") for index in range(1, 11)]
        outputs = [make_output(case["case_id"]) for case in cases]
        outputs[0]["decision"] = "reject"
        outputs[1]["decision"] = "reject"
        result = evaluate.evaluate_records(cases, outputs, "candidate_block_rate")
        self.assertEqual(result["decision"], "BLOCK")
        self.assertEqual(result["metrics"]["pass_rate"], 0.8)
        self.assertEqual(result["metrics"]["critical_failure_count"], 0)

    def test_block_for_any_critical_failure(self) -> None:
        cases = [make_case(f"c{index}") for index in range(1, 11)]
        cases[0]["severity"] = "critical"
        outputs = [make_output(case["case_id"]) for case in cases]
        outputs[0]["decision"] = "reject"
        result = evaluate.evaluate_records(cases, outputs, "candidate_block_critical")
        self.assertEqual(result["metrics"]["pass_rate"], 0.9)
        self.assertEqual(result["metrics"]["critical_failure_count"], 1)
        self.assertEqual(result["decision"], "BLOCK")

    def test_all_checks_for_tool_arguments_citations_and_forbidden_tools(self) -> None:
        case = make_case(
            "tool-case",
            decision="approve",
            tool_name="create_reimbursement",
            required_args={"claim": {"amount": 50}},
            required_citations=["policy-4"],
            forbidden_tools=["reject_claim"],
        )
        output = make_output(
            "tool-case",
            tool_calls=[
                {
                    "name": "create_reimbursement",
                    "arguments": {
                        "claim": {"amount": 50, "currency": "USD"},
                        "audit": True,
                    },
                }
            ],
            citations=["policy-4", "receipt-8"],
        )
        result = evaluate.evaluate_records([case], [output], "tool_pass")
        self.assertTrue(result["cases"][0]["passed"])
        self.assertTrue(all(result["cases"][0]["checks"].values()))


class ValidationTests(unittest.TestCase):
    def test_malformed_json_fails_loudly(self) -> None:
        with tempfile.TemporaryDirectory() as temp_dir:
            path = Path(temp_dir) / "bad.jsonl"
            path.write_text('{"case_id": "c1"\n', encoding="utf-8")
            with self.assertRaisesRegex(evaluate.EvaluationError, "malformed JSON"):
                evaluate.load_cases(path)

    def test_duplicate_output_ids_fail(self) -> None:
        with tempfile.TemporaryDirectory() as temp_dir:
            path = Path(temp_dir) / "outputs.jsonl"
            output = make_output("c1")
            path.write_text(
                json.dumps(output) + "\n" + json.dumps(output) + "\n",
                encoding="utf-8",
            )
            with self.assertRaisesRegex(evaluate.EvaluationError, "duplicate case_id"):
                evaluate.load_outputs(path)

    def test_missing_and_extra_ids_fail(self) -> None:
        cases = [make_case("wanted")]
        outputs = [make_output("unexpected")]
        with self.assertRaisesRegex(
            evaluate.EvaluationError, "missing output case IDs: wanted"
        ):
            evaluate.evaluate_records(cases, outputs, "id_mismatch")

    def test_invalid_severity_fails(self) -> None:
        with tempfile.TemporaryDirectory() as temp_dir:
            path = Path(temp_dir) / "cases.jsonl"
            case = make_case("c1", severity="high")
            path.write_text(json.dumps(case) + "\n", encoding="utf-8")
            with self.assertRaisesRegex(evaluate.EvaluationError, "unsupported value"):
                evaluate.load_cases(path)


if __name__ == "__main__":
    unittest.main()