kplam
51dc8dc4f6
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
100 lines
4.9 KiB
Python
100 lines
4.9 KiB
Python
"""LLM-as-judge 评测指标单元测试（FakeOllama 替身，不依赖真实模型）"""
|
|
|
|
import pytest
|
|
|
|
from scripts.eval.judge import hallucination_rate, taxonomy_consistency
|
|
|
|
|
|
class FakeOllama:
|
|
"""记录 prompt 并按序返回预设响应（或抛出预设异常）的 OllamaClient 替身"""
|
|
|
|
def __init__(self, responses: list[str] | None = None, error: Exception | None = None) -> None:
|
|
self.prompts: list[str] = []
|
|
self.json_modes: list[bool] = []
|
|
self._responses = list(responses or [])
|
|
self._error = error
|
|
|
|
async def generate(self, prompt: str, json_mode: bool = False) -> str:
|
|
self.prompts.append(prompt)
|
|
self.json_modes.append(json_mode)
|
|
if self._error is not None:
|
|
raise self._error
|
|
return self._responses.pop(0)
|
|
|
|
|
|
def _assertions_json(*supported: bool) -> str:
|
|
"""构造 hallucination 判定的 JSON 输出，每个 bool 对应一条断言的 supported"""
|
|
claims = ", ".join(f'{{"claim": "断言{i}", "supported": {str(flag).lower()}}}' for i, flag in enumerate(supported))
|
|
return f'{{"assertions": [{claims}]}}'
|
|
|
|
|
|
class TestHallucinationRate:
|
|
async def test_all_supported_returns_zero(self) -> None:
|
|
ollama = FakeOllama([_assertions_json(True, True, True)])
|
|
assert await hallucination_rate("摘要", "原文", ollama) == 0.0 # type: ignore[arg-type]
|
|
|
|
async def test_prompt_contains_source_and_summary(self) -> None:
|
|
ollama = FakeOllama([_assertions_json(True)])
|
|
await hallucination_rate("这是摘要内容", "这是原文内容", ollama) # type: ignore[arg-type]
|
|
assert len(ollama.prompts) == 1
|
|
assert "这是摘要内容" in ollama.prompts[0]
|
|
assert "这是原文内容" in ollama.prompts[0]
|
|
assert ollama.json_modes == [True]
|
|
|
|
async def test_two_of_five_unsupported(self) -> None:
|
|
ollama = FakeOllama([_assertions_json(True, False, True, False, True)])
|
|
rate = await hallucination_rate("摘要", "原文", ollama) # type: ignore[arg-type]
|
|
assert rate == pytest.approx(0.4)
|
|
|
|
async def test_non_json_output_returns_zero(self) -> None:
|
|
ollama = FakeOllama(["我无法完成这个判定任务。"])
|
|
assert await hallucination_rate("摘要", "原文", ollama) == 0.0 # type: ignore[arg-type]
|
|
|
|
async def test_missing_assertions_field_returns_zero(self) -> None:
|
|
ollama = FakeOllama(['{"result": "ok"}'])
|
|
assert await hallucination_rate("摘要", "原文", ollama) == 0.0 # type: ignore[arg-type]
|
|
|
|
async def test_generate_error_returns_zero(self) -> None:
|
|
ollama = FakeOllama(error=RuntimeError("Ollama 不可用"))
|
|
assert await hallucination_rate("摘要", "原文", ollama) == 0.0 # type: ignore[arg-type]
|
|
|
|
async def test_noisy_output_json_extracted(self) -> None:
|
|
# 模型输出前后带噪声时仍能提取首个 JSON 对象
|
|
ollama = FakeOllama([f"先分析一下。\n{_assertions_json(True, False)}\n以上。"])
|
|
rate = await hallucination_rate("摘要", "原文", ollama) # type: ignore[arg-type]
|
|
assert rate == pytest.approx(0.5)
|
|
|
|
async def test_claims_truncated_to_max(self) -> None:
|
|
# 超过 _MAX_CLAIMS(5) 的断言不计入：前 5 条全支持，第 6/7 条不支持 → 0.0
|
|
ollama = FakeOllama([_assertions_json(True, True, True, True, True, False, False)])
|
|
assert await hallucination_rate("摘要", "原文", ollama) == 0.0 # type: ignore[arg-type]
|
|
|
|
|
|
class TestTaxonomyConsistency:
|
|
async def test_consistent_true(self) -> None:
|
|
ollama = FakeOllama(['{"consistent": true}'])
|
|
assert await taxonomy_consistency("摘要", "制度", ollama) is True # type: ignore[arg-type]
|
|
|
|
async def test_consistent_false(self) -> None:
|
|
ollama = FakeOllama(['{"consistent": false}'])
|
|
assert await taxonomy_consistency("摘要", "制度", ollama) is False # type: ignore[arg-type]
|
|
|
|
async def test_prompt_contains_category_and_summary(self) -> None:
|
|
ollama = FakeOllama(['{"consistent": true}'])
|
|
await taxonomy_consistency("这是摘要内容", "人事制度", ollama) # type: ignore[arg-type]
|
|
assert "人事制度" in ollama.prompts[0]
|
|
assert "这是摘要内容" in ollama.prompts[0]
|
|
assert ollama.json_modes == [True]
|
|
|
|
async def test_non_json_output_returns_default_true(self) -> None:
|
|
ollama = FakeOllama(["无法判定"])
|
|
assert await taxonomy_consistency("摘要", "制度", ollama) is True # type: ignore[arg-type]
|
|
|
|
async def test_missing_consistent_field_returns_default_true(self) -> None:
|
|
ollama = FakeOllama(['{"ok": 1}'])
|
|
assert await taxonomy_consistency("摘要", "制度", ollama) is True # type: ignore[arg-type]
|
|
|
|
async def test_generate_error_returns_default_true(self) -> None:
|
|
ollama = FakeOllama(error=RuntimeError("Ollama 不可用"))
|
|
assert await taxonomy_consistency("摘要", "制度", ollama) is True # type: ignore[arg-type]
|