kplam
51dc8dc4f6
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
132 lines
4.3 KiB
Python
132 lines
4.3 KiB
Python
"""run_eval 门槛常量与报告组装纯函数单元测试"""
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from scripts.eval.run_eval import (
|
|
THRESHOLD_HALLUCINATION,
|
|
THRESHOLD_L1_ER,
|
|
THRESHOLD_L3_ER,
|
|
THRESHOLD_PRUNING_LOSS,
|
|
_build_report,
|
|
_fmt,
|
|
_mark,
|
|
)
|
|
|
|
|
|
class TestThresholds:
|
|
"""Spec 规定的门槛值：L1 ER ≥ 0.85 / L3 ER ≥ 0.9 / 幻觉率 < 2% / Pruning Loss < 8%"""
|
|
|
|
def test_l1_entity_recall_threshold(self) -> None:
|
|
assert THRESHOLD_L1_ER == 0.85
|
|
|
|
def test_l3_entity_recall_threshold(self) -> None:
|
|
assert THRESHOLD_L3_ER == 0.9
|
|
|
|
def test_hallucination_threshold(self) -> None:
|
|
assert THRESHOLD_HALLUCINATION == 0.02
|
|
|
|
def test_pruning_loss_threshold(self) -> None:
|
|
assert THRESHOLD_PRUNING_LOSS == 0.08
|
|
|
|
|
|
class TestMark:
|
|
def test_pass(self) -> None:
|
|
assert _mark(True) == "✅"
|
|
|
|
def test_fail(self) -> None:
|
|
assert _mark(False) == "❌"
|
|
|
|
|
|
class TestFmt:
|
|
def test_none_returns_na(self) -> None:
|
|
assert _fmt(None) == "N/A"
|
|
assert _fmt(None, percent=True) == "N/A"
|
|
|
|
def test_decimal_format(self) -> None:
|
|
assert _fmt(0.85) == "0.8500"
|
|
|
|
def test_percent_format(self) -> None:
|
|
assert _fmt(0.1234, percent=True) == "12.3%"
|
|
assert _fmt(0.02, percent=True) == "2.0%"
|
|
|
|
|
|
def _doc_record(**overrides: Any) -> dict:
|
|
record: dict[str, Any] = {
|
|
"id": "d1",
|
|
"title": "考勤制度",
|
|
"golden_category": "制度",
|
|
"category": "制度",
|
|
"entity_recall_l1": 0.9,
|
|
"entity_recall_l3": 0.95,
|
|
"hallucination_rate": 0.01,
|
|
"taxonomy_consistency": True,
|
|
}
|
|
record.update(overrides)
|
|
return record
|
|
|
|
|
|
def _report_kwargs(**overrides: Any) -> dict:
|
|
kwargs: dict[str, Any] = {
|
|
"regression_path": Path("regression_set.json"),
|
|
"doc_records": [_doc_record()],
|
|
"query_summary": {"total": 4, "positive": 3, "negative": 1, "negative_false_alarm": 0},
|
|
"hier_metrics": {"precision@5": 0.6, "recall@10": 0.8},
|
|
"baseline_metrics": {"precision@5": 0.4, "recall@10": 0.7},
|
|
"routing": {"precision": 0.75, "recall": 0.8, "f1": 0.77},
|
|
"prune_loss": 0.05,
|
|
"judge_enabled": True,
|
|
"kept_data": False,
|
|
}
|
|
kwargs.update(overrides)
|
|
return kwargs
|
|
|
|
|
|
class TestBuildReport:
|
|
def test_baseline_comparison_columns(self) -> None:
|
|
report = _build_report(**_report_kwargs())
|
|
assert "| 指标 | 分层检索 | 平铺 baseline |" in report
|
|
assert "| Precision@5 | 0.6000 | 0.4000 |" in report
|
|
assert "| Recall@10 | 0.8000 | 0.7000 |" in report
|
|
|
|
def test_doc_table_row(self) -> None:
|
|
report = _build_report(**_report_kwargs())
|
|
assert "| d1 | 考勤制度 | 制度 | 制度 | 0.9000 | 0.9500 | 1.0% | 是 |" in report
|
|
|
|
def test_threshold_pass_marks(self) -> None:
|
|
report = _build_report(**_report_kwargs())
|
|
assert "| L1 Entity Recall | 0.9000 | ≥ 0.85 | ✅ |" in report
|
|
assert "| L3 Entity Recall | 0.9500 | ≥ 0.9 | ✅ |" in report
|
|
assert "| Hallucination Rate | 1.0% | < 2% | ✅ |" in report
|
|
assert "| Pruning Loss | 5.0% | < 8% | ✅ |" in report
|
|
|
|
def test_threshold_fail_marks(self) -> None:
|
|
report = _build_report(
|
|
**_report_kwargs(
|
|
doc_records=[
|
|
_doc_record(
|
|
entity_recall_l1=0.8,
|
|
entity_recall_l3=0.7,
|
|
hallucination_rate=0.05,
|
|
taxonomy_consistency=False,
|
|
)
|
|
],
|
|
prune_loss=0.2,
|
|
)
|
|
)
|
|
assert "| L1 Entity Recall | 0.8000 | ≥ 0.85 | ❌ |" in report
|
|
assert "| L3 Entity Recall | 0.7000 | ≥ 0.9 | ❌ |" in report
|
|
assert "| Hallucination Rate | 5.0% | < 2% | ❌ |" in report
|
|
assert "| Pruning Loss | 20.0% | < 8% | ❌ |" in report
|
|
assert " | 否 |" in report
|
|
|
|
def test_judge_skipped_renders_na(self) -> None:
|
|
report = _build_report(
|
|
**_report_kwargs(
|
|
doc_records=[_doc_record(hallucination_rate=None, taxonomy_consistency=None)],
|
|
judge_enabled=False,
|
|
)
|
|
)
|
|
assert "N/A" in report
|
|
assert "跳过" in report
|