Initial commit: QMDSearch 分层信息检索服务
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
This commit is contained in:
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""分层摘要 RAG 离线评测 harness
|
||||
|
||||
流程：回归集 → 独立 _eval 后缀集合入库（Ingester）→ 摘要质量指标（Entity Recall /
|
||||
幻觉率 / 类目一致性）→ 检索效用指标（Routing F1 / Pruning Loss / Precision@5 /
|
||||
Recall@10）→ 平铺 chunks baseline 对比 → Markdown 报告（stdout + report.md）。
|
||||
|
||||
用法：
|
||||
uv run python scripts/eval/run_eval.py [--regression PATH] [--keep-data] [--no-judge]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 脚本直运行时把项目根加入 sys.path，保证可以 import app 与 scripts.eval
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
import structlog # noqa: E402
|
||||
|
||||
from app.config import settings # noqa: E402
|
||||
from app.core import ingestion as ingestion_mod # noqa: E402
|
||||
from app.core import retriever as retriever_mod # noqa: E402
|
||||
from app.core.ingestion import Ingester # noqa: E402
|
||||
from app.core.retriever import Retriever # noqa: E402
|
||||
from app.models.document import DocumentInput # noqa: E402
|
||||
from app.models.search import SearchRequest # noqa: E402
|
||||
from app.services import qdrant as qdrant_mod # noqa: E402
|
||||
from app.services.ollama import OllamaClient # noqa: E402
|
||||
from app.services.qdrant import QdrantService # noqa: E402
|
||||
from scripts.eval.judge import hallucination_rate, taxonomy_consistency # noqa: E402
|
||||
from scripts.eval.metrics import ( # noqa: E402
|
||||
aggregate,
|
||||
entity_recall,
|
||||
precision_at_k,
|
||||
pruning_loss,
|
||||
recall_at_k,
|
||||
routing_f1,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 评测集合后缀与默认路径
|
||||
EVAL_SUFFIX = "_eval"
|
||||
_SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_REGRESSION = _SCRIPT_DIR / "regression_set.json"
|
||||
REPORT_PATH = _SCRIPT_DIR / "report.md"
|
||||
|
||||
# 门槛（Spec 规定）：低于/高于门槛在报告中标红
|
||||
THRESHOLD_L1_ER = 0.85 # L1 Entity Recall ≥ 0.85
|
||||
THRESHOLD_L3_ER = 0.9 # L3 Entity Recall ≥ 0.9
|
||||
THRESHOLD_HALLUCINATION = 0.02 # 幻觉率 < 2%
|
||||
THRESHOLD_PRUNING_LOSS = 0.08 # Pruning Loss < 8%
|
||||
|
||||
|
||||
def _switch_to_eval_collections() -> list[str]:
|
||||
"""把 app 内模块级集合名常量整体切换为 _eval 后缀的评测集合
|
||||
|
||||
QdrantService / Retriever / Ingester 均在各自模块命名空间引用了集合名常量，
|
||||
评测脚本统一改写这些模块属性实现集合隔离，不改动 app 源码。
|
||||
返回评测集合名列表（用于评测结束后清理）。
|
||||
"""
|
||||
eval_names = {
|
||||
"COLLECTION_L1": f"{qdrant_mod.COLLECTION_L1}{EVAL_SUFFIX}",
|
||||
"COLLECTION_L2": f"{qdrant_mod.COLLECTION_L2}{EVAL_SUFFIX}",
|
||||
"COLLECTION_L3": f"{qdrant_mod.COLLECTION_L3}{EVAL_SUFFIX}",
|
||||
"COLLECTION_CHUNKS": f"{qdrant_mod.COLLECTION_CHUNKS}{EVAL_SUFFIX}",
|
||||
}
|
||||
for module in (qdrant_mod, retriever_mod, ingestion_mod):
|
||||
for name, value in eval_names.items():
|
||||
if hasattr(module, name):
|
||||
setattr(module, name, value)
|
||||
qdrant_mod.ALL_COLLECTIONS = tuple(eval_names.values())
|
||||
sparse_eval = (eval_names["COLLECTION_L1"], eval_names["COLLECTION_CHUNKS"])
|
||||
qdrant_mod.SPARSE_COLLECTIONS = sparse_eval
|
||||
retriever_mod.SPARSE_COLLECTIONS = sparse_eval
|
||||
# upsert_nodes 按集合名校验层级前缀，需同步替换
|
||||
qdrant_mod._NODE_ID_PREFIX = {eval_names["COLLECTION_L2"]: "l2", eval_names["COLLECTION_L3"]: "l3"}
|
||||
return list(eval_names.values())
|
||||
|
||||
|
||||
async def _check_services(qdrant: QdrantService, ollama: OllamaClient) -> str | None:
|
||||
"""检查 Qdrant / Ollama 连通性，返回错误消息（None 表示正常）"""
|
||||
try:
|
||||
await qdrant.client.get_collections()
|
||||
except Exception as exc:
|
||||
return f"无法连接 Qdrant（{settings.qdrant_host}:{settings.qdrant_port}）：{exc}"
|
||||
if not await ollama.is_available():
|
||||
return f"无法连接 Ollama（{settings.ollama_base_url}），请确认服务已启动（入库与评测均依赖 Ollama）"
|
||||
return None
|
||||
|
||||
|
||||
async def _l1_candidate_doc_ids(retriever: Retriever, query: str) -> set[str]:
|
||||
"""轻量复现 Retriever 的 L1 路由层：embed → L1 集合 top-N → 候选 doc_id 集合
|
||||
|
||||
用于计算 Pruning Loss 与 Routing F1；不套类目过滤，度量纯 L1 向量召回。
|
||||
"""
|
||||
dense = (await retriever.embedding.embed([query]))[0]
|
||||
sparse = retriever.sparse_encoder.encode(query) if settings.sparse_enabled else None
|
||||
# 复用 Retriever 内部检索方法（hybrid/dense 按集合能力自动选择）
|
||||
hits = await retriever._search_collection(retriever_mod.COLLECTION_L1, dense, sparse, settings.l1_doc_top_n, None)
|
||||
return {(p.payload or {}).get("doc_id", "") for p in hits} - {""}
|
||||
|
||||
|
||||
async def _baseline_doc_ids(retriever: Retriever, query: str) -> list[str]:
|
||||
"""平铺 baseline：直接对 chunks 集合做 hybrid top-k（无路由无剪枝），返回命中 doc_id 列表"""
|
||||
dense = (await retriever.embedding.embed([query]))[0]
|
||||
sparse = retriever.sparse_encoder.encode(query) if settings.sparse_enabled else None
|
||||
points = await retriever._search_collection(
|
||||
retriever_mod.COLLECTION_CHUNKS, dense, sparse, settings.retrieval_top_k, None
|
||||
)
|
||||
return [(p.payload or {}).get("doc_id", "") for p in points]
|
||||
|
||||
|
||||
def _mark(ok: bool) -> str:
|
||||
"""门槛判定标记"""
|
||||
return "✅" if ok else "❌"
|
||||
|
||||
|
||||
def _fmt(value: float | None, percent: bool = False) -> str:
|
||||
"""数值格式化；None 表示未评测（judge 跳过）"""
|
||||
if value is None:
|
||||
return "N/A"
|
||||
return f"{value:.1%}" if percent else f"{value:.4f}"
|
||||
|
||||
|
||||
def _build_report(
|
||||
regression_path: Path,
|
||||
doc_records: list[dict],
|
||||
query_summary: dict,
|
||||
hier_metrics: dict[str, float],
|
||||
baseline_metrics: dict[str, float],
|
||||
routing: dict[str, float],
|
||||
prune_loss: float,
|
||||
judge_enabled: bool,
|
||||
kept_data: bool,
|
||||
) -> str:
|
||||
"""组装 Markdown 评测报告"""
|
||||
lines: list[str] = [
|
||||
"# 分层摘要 RAG 评测报告",
|
||||
"",
|
||||
f"- 回归集：`{regression_path}`",
|
||||
f"- 文档数：{len(doc_records)}；query 数：{query_summary['total']}"
|
||||
f"（positive {query_summary['positive']} / negative {query_summary['negative']}）",
|
||||
f"- LLM judge：{'开启' if judge_enabled else '跳过（--no-judge 或 Ollama 不可用）'}",
|
||||
f"- 评测集合：`*{EVAL_SUFFIX}`（{'保留' if kept_data else '已清理'}）",
|
||||
"",
|
||||
"## 摘要质量（按文档）",
|
||||
"",
|
||||
"| 文档 | 标题 | golden 类目 | 实际类目 | L1 Entity Recall | L3 Entity Recall | 幻觉率 | 类目一致 |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for record in doc_records:
|
||||
consistency = record.get("taxonomy_consistency")
|
||||
lines.append(
|
||||
f"| {record['id']} | {record['title']} | {record['golden_category']} | {record['category']} "
|
||||
f"| {_fmt(record['entity_recall_l1'])} | {_fmt(record['entity_recall_l3'])} "
|
||||
f"| {_fmt(record.get('hallucination_rate'), percent=True)} "
|
||||
f"| {('是' if consistency else '否') if consistency is not None else 'N/A'} |"
|
||||
)
|
||||
|
||||
# 汇总与门槛判定
|
||||
er_l1 = aggregate([{"v": r["entity_recall_l1"]} for r in doc_records]).get("v", 0.0)
|
||||
er_l3 = aggregate([{"v": r["entity_recall_l3"]} for r in doc_records]).get("v", 0.0)
|
||||
hall_records = [{"v": r["hallucination_rate"]} for r in doc_records if r.get("hallucination_rate") is not None]
|
||||
hall = aggregate(hall_records).get("v") if hall_records else None
|
||||
lines += [
|
||||
"",
|
||||
"## 指标汇总与门槛判定",
|
||||
"",
|
||||
"| 指标 | 数值 | 门槛 | 判定 |",
|
||||
"| --- | --- | --- | --- |",
|
||||
f"| L1 Entity Recall | {_fmt(er_l1)} | ≥ {THRESHOLD_L1_ER} | {_mark(er_l1 >= THRESHOLD_L1_ER)} |",
|
||||
f"| L3 Entity Recall | {_fmt(er_l3)} | ≥ {THRESHOLD_L3_ER} | {_mark(er_l3 >= THRESHOLD_L3_ER)} |",
|
||||
f"| Hallucination Rate | {_fmt(hall, percent=True)} | < {THRESHOLD_HALLUCINATION:.0%} "
|
||||
f"| {_mark(hall < THRESHOLD_HALLUCINATION) if hall is not None else 'N/A'} |",
|
||||
f"| Pruning Loss | {_fmt(prune_loss, percent=True)} | < {THRESHOLD_PRUNING_LOSS:.0%} "
|
||||
f"| {_mark(prune_loss < THRESHOLD_PRUNING_LOSS)} |",
|
||||
"",
|
||||
"## 检索效用（hierarchical vs 平铺 baseline）",
|
||||
"",
|
||||
"| 指标 | 分层检索 | 平铺 baseline |",
|
||||
"| --- | --- | --- |",
|
||||
f"| Precision@5 | {_fmt(hier_metrics.get('precision@5', 0.0))} "
|
||||
f"| {_fmt(baseline_metrics.get('precision@5', 0.0))} |",
|
||||
f"| Recall@10 | {_fmt(hier_metrics.get('recall@10', 0.0))} | {_fmt(baseline_metrics.get('recall@10', 0.0))} |",
|
||||
"",
|
||||
"### 路由层（L1）",
|
||||
"",
|
||||
f"- Routing Precision：{_fmt(routing['precision'])}",
|
||||
f"- Routing Recall：{_fmt(routing['recall'])}",
|
||||
f"- Routing F1：{_fmt(routing['f1'])}",
|
||||
f"- Pruning Loss：{_fmt(prune_loss, percent=True)}（{_mark(prune_loss < THRESHOLD_PRUNING_LOSS)}）",
|
||||
"",
|
||||
"### negative query",
|
||||
"",
|
||||
f"- 误中（返回了任意结果）：{query_summary['negative_false_alarm']} / {query_summary['negative']}",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def run(regression_path: Path, keep_data: bool, judge_enabled: bool) -> int:
|
||||
"""评测主流程，返回进程退出码"""
|
||||
data = json.loads(regression_path.read_text(encoding="utf-8"))
|
||||
documents: list[dict] = data["documents"]
|
||||
queries: list[dict] = data["queries"]
|
||||
|
||||
eval_collections = _switch_to_eval_collections()
|
||||
qdrant = QdrantService()
|
||||
ollama = OllamaClient()
|
||||
|
||||
# 连通性检查：失败给出中文提示并以退出码 2 结束
|
||||
error = await _check_services(qdrant, ollama)
|
||||
if error:
|
||||
print(f"错误：{error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
await qdrant.ensure_collections()
|
||||
logger.info("评测集合就绪", collections=eval_collections)
|
||||
|
||||
ingester = Ingester(qdrant=qdrant)
|
||||
retriever = Retriever(qdrant=qdrant)
|
||||
|
||||
id_map: dict[str, str] = {} # 回归集 doc id -> 实际入库 doc_id
|
||||
doc_records: list[dict] = []
|
||||
hier_records: list[dict[str, float]] = []
|
||||
baseline_records: list[dict[str, float]] = []
|
||||
pruned_cases: list[bool] = []
|
||||
golden_sets: list[set[str]] = []
|
||||
routed_sets: list[set[str]] = []
|
||||
negative_total = 0
|
||||
negative_false_alarm = 0
|
||||
|
||||
try:
|
||||
# 1. 逐篇入库并计算摘要质量指标
|
||||
for doc in documents:
|
||||
result = await ingester.ingest(DocumentInput(title=doc["title"], text=doc["text"]))
|
||||
id_map[doc["id"]] = result.document_id
|
||||
logger.info("文档入库完成", id=doc["id"], doc_id=result.document_id, category=result.category)
|
||||
|
||||
record: dict = {
|
||||
"id": doc["id"],
|
||||
"title": doc["title"],
|
||||
"golden_category": doc["golden_category"],
|
||||
"category": result.category,
|
||||
"entity_recall_l1": entity_recall(doc["text"], result.summary.l1_summary),
|
||||
"entity_recall_l3": entity_recall(doc["text"], result.summary.l3_content_outline),
|
||||
"hallucination_rate": None,
|
||||
"taxonomy_consistency": None,
|
||||
}
|
||||
if judge_enabled:
|
||||
record["hallucination_rate"] = await hallucination_rate(result.summary.l1_summary, doc["text"], ollama)
|
||||
record["taxonomy_consistency"] = await taxonomy_consistency(
|
||||
result.summary.l1_summary, result.category, ollama
|
||||
)
|
||||
doc_records.append(record)
|
||||
|
||||
# 2. 逐 query 计算检索效用指标（分层检索 + 轻量 L1 路由层 + 平铺 baseline）
|
||||
for query in queries:
|
||||
query_text = query["query"]
|
||||
golden_ids = {id_map[query["golden_doc_id"]]} if query.get("golden_doc_id") else set()
|
||||
|
||||
response = await retriever.search(SearchRequest(query=query_text))
|
||||
hit_doc_ids = [hit.doc_id for hit in response.hits]
|
||||
l1_candidates = await _l1_candidate_doc_ids(retriever, query_text)
|
||||
|
||||
if query["type"] == "positive" and golden_ids:
|
||||
pruned_cases.append(not golden_ids & l1_candidates)
|
||||
golden_sets.append(golden_ids)
|
||||
routed_sets.append(l1_candidates)
|
||||
hier_records.append(
|
||||
{
|
||||
"precision@5": precision_at_k(hit_doc_ids, golden_ids, 5),
|
||||
"recall@10": recall_at_k(hit_doc_ids, golden_ids, 10),
|
||||
}
|
||||
)
|
||||
baseline_ids = await _baseline_doc_ids(retriever, query_text)
|
||||
baseline_records.append(
|
||||
{
|
||||
"precision@5": precision_at_k(baseline_ids, golden_ids, 5),
|
||||
"recall@10": recall_at_k(baseline_ids, golden_ids, 10),
|
||||
}
|
||||
)
|
||||
else:
|
||||
# negative query：期望无结果，任何返回均计为误中
|
||||
negative_total += 1
|
||||
if hit_doc_ids:
|
||||
negative_false_alarm += 1
|
||||
finally:
|
||||
# 评测集合清理（--keep-data 时保留）
|
||||
if keep_data:
|
||||
logger.info("--keep-data 生效，保留评测集合", collections=eval_collections)
|
||||
else:
|
||||
for collection in eval_collections:
|
||||
try:
|
||||
await qdrant.client.delete_collection(collection)
|
||||
except Exception as exc:
|
||||
logger.warning("评测集合清理失败", collection=collection, error=str(exc))
|
||||
logger.info("评测集合已清理", collections=eval_collections)
|
||||
|
||||
# 3. 汇总并输出报告
|
||||
prune_loss = pruning_loss(pruned_cases)
|
||||
routing = routing_f1(golden_sets, routed_sets)
|
||||
hier_metrics = aggregate(hier_records)
|
||||
baseline_metrics = aggregate(baseline_records)
|
||||
query_summary = {
|
||||
"total": len(queries),
|
||||
"positive": len(queries) - negative_total,
|
||||
"negative": negative_total,
|
||||
"negative_false_alarm": negative_false_alarm,
|
||||
}
|
||||
report = _build_report(
|
||||
regression_path,
|
||||
doc_records,
|
||||
query_summary,
|
||||
hier_metrics,
|
||||
baseline_metrics,
|
||||
routing,
|
||||
prune_loss,
|
||||
judge_enabled,
|
||||
keep_data,
|
||||
)
|
||||
print(report)
|
||||
REPORT_PATH.write_text(report + "\n", encoding="utf-8")
|
||||
logger.info("评测报告已写入", path=str(REPORT_PATH))
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="分层摘要 RAG 离线评测：回归集入库 → 摘要质量/检索效用指标 → 平铺 baseline 对比 → Markdown 报告",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--regression",
|
||||
type=Path,
|
||||
default=DEFAULT_REGRESSION,
|
||||
help=f"回归集 JSON 路径（默认 {DEFAULT_REGRESSION}）",
|
||||
)
|
||||
parser.add_argument("--keep-data", action="store_true", help="评测结束后保留 _eval 集合（默认删除）")
|
||||
parser.add_argument("--no-judge", action="store_true", help="跳过 LLM-as-judge 指标（幻觉率 / 类目一致性）")
|
||||
args = parser.parse_args()
|
||||
return asyncio.run(run(args.regression.resolve(), args.keep_data, not args.no_judge))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user