"""LLM-as-judge 评测指标单元测试（FakeOllama 替身，不依赖真实模型）"""

import pytest

from scripts.eval.judge import hallucination_rate, taxonomy_consistency


class FakeOllama:
    """记录 prompt 并按序返回预设响应（或抛出预设异常）的 OllamaClient 替身"""

    def __init__(self, responses: list[str] | None = None, error: Exception | None = None) -> None:
        self.prompts: list[str] = []
        self.json_modes: list[bool] = []
        self._responses = list(responses or [])
        self._error = error

    async def generate(self, prompt: str, json_mode: bool = False) -> str:
        self.prompts.append(prompt)
        self.json_modes.append(json_mode)
        if self._error is not None:
            raise self._error
        return self._responses.pop(0)


def _assertions_json(*supported: bool) -> str:
    """构造 hallucination 判定的 JSON 输出，每个 bool 对应一条断言的 supported"""
    claims = ", ".join(f'{{"claim": "断言{i}", "supported": {str(flag).lower()}}}' for i, flag in enumerate(supported))
    return f'{{"assertions": [{claims}]}}'


class TestHallucinationRate:
    async def test_all_supported_returns_zero(self) -> None:
        ollama = FakeOllama([_assertions_json(True, True, True)])
        assert await hallucination_rate("摘要", "原文", ollama) == 0.0  # type: ignore[arg-type]

    async def test_prompt_contains_source_and_summary(self) -> None:
        ollama = FakeOllama([_assertions_json(True)])
        await hallucination_rate("这是摘要内容", "这是原文内容", ollama)  # type: ignore[arg-type]
        assert len(ollama.prompts) == 1
        assert "这是摘要内容" in ollama.prompts[0]
        assert "这是原文内容" in ollama.prompts[0]
        assert ollama.json_modes == [True]

    async def test_two_of_five_unsupported(self) -> None:
        ollama = FakeOllama([_assertions_json(True, False, True, False, True)])
        rate = await hallucination_rate("摘要", "原文", ollama)  # type: ignore[arg-type]
        assert rate == pytest.approx(0.4)

    async def test_non_json_output_returns_zero(self) -> None:
        ollama = FakeOllama(["我无法完成这个判定任务。"])
        assert await hallucination_rate("摘要", "原文", ollama) == 0.0  # type: ignore[arg-type]

    async def test_missing_assertions_field_returns_zero(self) -> None:
        ollama = FakeOllama(['{"result": "ok"}'])
        assert await hallucination_rate("摘要", "原文", ollama) == 0.0  # type: ignore[arg-type]

    async def test_generate_error_returns_zero(self) -> None:
        ollama = FakeOllama(error=RuntimeError("Ollama 不可用"))
        assert await hallucination_rate("摘要", "原文", ollama) == 0.0  # type: ignore[arg-type]

    async def test_noisy_output_json_extracted(self) -> None:
        # 模型输出前后带噪声时仍能提取首个 JSON 对象
        ollama = FakeOllama([f"先分析一下。\n{_assertions_json(True, False)}\n以上。"])
        rate = await hallucination_rate("摘要", "原文", ollama)  # type: ignore[arg-type]
        assert rate == pytest.approx(0.5)

    async def test_claims_truncated_to_max(self) -> None:
        # 超过 _MAX_CLAIMS(5) 的断言不计入：前 5 条全支持，第 6/7 条不支持 → 0.0
        ollama = FakeOllama([_assertions_json(True, True, True, True, True, False, False)])
        assert await hallucination_rate("摘要", "原文", ollama) == 0.0  # type: ignore[arg-type]


class TestTaxonomyConsistency:
    async def test_consistent_true(self) -> None:
        ollama = FakeOllama(['{"consistent": true}'])
        assert await taxonomy_consistency("摘要", "制度", ollama) is True  # type: ignore[arg-type]

    async def test_consistent_false(self) -> None:
        ollama = FakeOllama(['{"consistent": false}'])
        assert await taxonomy_consistency("摘要", "制度", ollama) is False  # type: ignore[arg-type]

    async def test_prompt_contains_category_and_summary(self) -> None:
        ollama = FakeOllama(['{"consistent": true}'])
        await taxonomy_consistency("这是摘要内容", "人事制度", ollama)  # type: ignore[arg-type]
        assert "人事制度" in ollama.prompts[0]
        assert "这是摘要内容" in ollama.prompts[0]
        assert ollama.json_modes == [True]

    async def test_non_json_output_returns_default_true(self) -> None:
        ollama = FakeOllama(["无法判定"])
        assert await taxonomy_consistency("摘要", "制度", ollama) is True  # type: ignore[arg-type]

    async def test_missing_consistent_field_returns_default_true(self) -> None:
        ollama = FakeOllama(['{"ok": 1}'])
        assert await taxonomy_consistency("摘要", "制度", ollama) is True  # type: ignore[arg-type]

    async def test_generate_error_returns_default_true(self) -> None:
        ollama = FakeOllama(error=RuntimeError("Ollama 不可用"))
        assert await taxonomy_consistency("摘要", "制度", ollama) is True  # type: ignore[arg-type]