Initial commit: QMDSearch 分层信息检索服务
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索(dense + sparse RRF 融合) - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
"""run_eval 门槛常量与报告组装纯函数单元测试"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scripts.eval.run_eval import (
|
||||
THRESHOLD_HALLUCINATION,
|
||||
THRESHOLD_L1_ER,
|
||||
THRESHOLD_L3_ER,
|
||||
THRESHOLD_PRUNING_LOSS,
|
||||
_build_report,
|
||||
_fmt,
|
||||
_mark,
|
||||
)
|
||||
|
||||
|
||||
class TestThresholds:
|
||||
"""Spec 规定的门槛值:L1 ER ≥ 0.85 / L3 ER ≥ 0.9 / 幻觉率 < 2% / Pruning Loss < 8%"""
|
||||
|
||||
def test_l1_entity_recall_threshold(self) -> None:
|
||||
assert THRESHOLD_L1_ER == 0.85
|
||||
|
||||
def test_l3_entity_recall_threshold(self) -> None:
|
||||
assert THRESHOLD_L3_ER == 0.9
|
||||
|
||||
def test_hallucination_threshold(self) -> None:
|
||||
assert THRESHOLD_HALLUCINATION == 0.02
|
||||
|
||||
def test_pruning_loss_threshold(self) -> None:
|
||||
assert THRESHOLD_PRUNING_LOSS == 0.08
|
||||
|
||||
|
||||
class TestMark:
|
||||
def test_pass(self) -> None:
|
||||
assert _mark(True) == "✅"
|
||||
|
||||
def test_fail(self) -> None:
|
||||
assert _mark(False) == "❌"
|
||||
|
||||
|
||||
class TestFmt:
|
||||
def test_none_returns_na(self) -> None:
|
||||
assert _fmt(None) == "N/A"
|
||||
assert _fmt(None, percent=True) == "N/A"
|
||||
|
||||
def test_decimal_format(self) -> None:
|
||||
assert _fmt(0.85) == "0.8500"
|
||||
|
||||
def test_percent_format(self) -> None:
|
||||
assert _fmt(0.1234, percent=True) == "12.3%"
|
||||
assert _fmt(0.02, percent=True) == "2.0%"
|
||||
|
||||
|
||||
def _doc_record(**overrides: Any) -> dict:
|
||||
record: dict[str, Any] = {
|
||||
"id": "d1",
|
||||
"title": "考勤制度",
|
||||
"golden_category": "制度",
|
||||
"category": "制度",
|
||||
"entity_recall_l1": 0.9,
|
||||
"entity_recall_l3": 0.95,
|
||||
"hallucination_rate": 0.01,
|
||||
"taxonomy_consistency": True,
|
||||
}
|
||||
record.update(overrides)
|
||||
return record
|
||||
|
||||
|
||||
def _report_kwargs(**overrides: Any) -> dict:
|
||||
kwargs: dict[str, Any] = {
|
||||
"regression_path": Path("regression_set.json"),
|
||||
"doc_records": [_doc_record()],
|
||||
"query_summary": {"total": 4, "positive": 3, "negative": 1, "negative_false_alarm": 0},
|
||||
"hier_metrics": {"precision@5": 0.6, "recall@10": 0.8},
|
||||
"baseline_metrics": {"precision@5": 0.4, "recall@10": 0.7},
|
||||
"routing": {"precision": 0.75, "recall": 0.8, "f1": 0.77},
|
||||
"prune_loss": 0.05,
|
||||
"judge_enabled": True,
|
||||
"kept_data": False,
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return kwargs
|
||||
|
||||
|
||||
class TestBuildReport:
|
||||
def test_baseline_comparison_columns(self) -> None:
|
||||
report = _build_report(**_report_kwargs())
|
||||
assert "| 指标 | 分层检索 | 平铺 baseline |" in report
|
||||
assert "| Precision@5 | 0.6000 | 0.4000 |" in report
|
||||
assert "| Recall@10 | 0.8000 | 0.7000 |" in report
|
||||
|
||||
def test_doc_table_row(self) -> None:
|
||||
report = _build_report(**_report_kwargs())
|
||||
assert "| d1 | 考勤制度 | 制度 | 制度 | 0.9000 | 0.9500 | 1.0% | 是 |" in report
|
||||
|
||||
def test_threshold_pass_marks(self) -> None:
|
||||
report = _build_report(**_report_kwargs())
|
||||
assert "| L1 Entity Recall | 0.9000 | ≥ 0.85 | ✅ |" in report
|
||||
assert "| L3 Entity Recall | 0.9500 | ≥ 0.9 | ✅ |" in report
|
||||
assert "| Hallucination Rate | 1.0% | < 2% | ✅ |" in report
|
||||
assert "| Pruning Loss | 5.0% | < 8% | ✅ |" in report
|
||||
|
||||
def test_threshold_fail_marks(self) -> None:
|
||||
report = _build_report(
|
||||
**_report_kwargs(
|
||||
doc_records=[
|
||||
_doc_record(
|
||||
entity_recall_l1=0.8,
|
||||
entity_recall_l3=0.7,
|
||||
hallucination_rate=0.05,
|
||||
taxonomy_consistency=False,
|
||||
)
|
||||
],
|
||||
prune_loss=0.2,
|
||||
)
|
||||
)
|
||||
assert "| L1 Entity Recall | 0.8000 | ≥ 0.85 | ❌ |" in report
|
||||
assert "| L3 Entity Recall | 0.7000 | ≥ 0.9 | ❌ |" in report
|
||||
assert "| Hallucination Rate | 5.0% | < 2% | ❌ |" in report
|
||||
assert "| Pruning Loss | 20.0% | < 8% | ❌ |" in report
|
||||
assert " | 否 |" in report
|
||||
|
||||
def test_judge_skipped_renders_na(self) -> None:
|
||||
report = _build_report(
|
||||
**_report_kwargs(
|
||||
doc_records=[_doc_record(hallucination_rate=None, taxonomy_consistency=None)],
|
||||
judge_enabled=False,
|
||||
)
|
||||
)
|
||||
assert "N/A" in report
|
||||
assert "跳过" in report
|
||||
Reference in New Issue
Block a user