Initial commit: QMDSearch 分层信息检索服务
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
This commit is contained in:
@@ -0,0 +1,93 @@
|
||||
"""LLM-as-judge 评测指标：基于 Ollama 小模型的幻觉率与类目一致性判定
|
||||
|
||||
所有判定均要求模型以 JSON 输出（json_mode），解析失败时保守处理并记录日志：
|
||||
- hallucination_rate 解析失败返回 0.0（不低报问题，避免阻塞评测主流程）
|
||||
- taxonomy_consistency 解析失败返回 True（不冤枉摘要）
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
import structlog
|
||||
|
||||
from app.services.ollama import OllamaClient
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 每次判定的断言抽取上限，避免长摘要导致判定过慢
|
||||
_MAX_CLAIMS = 5
|
||||
|
||||
_HALLUCINATION_PROMPT = """你是一个事实核查员。请从下面的【摘要】中抽取最多 {max_claims} 条事实性断言，
|
||||
并逐条判断【原文】是否支持该断言（断言中的数字、名称、条款等关键信息必须与原文一致才算支持）。
|
||||
|
||||
【原文】
|
||||
{source}
|
||||
|
||||
【摘要】
|
||||
{summary}
|
||||
|
||||
只输出 JSON，格式：{{"assertions": [{{"claim": "断言内容", "supported": true 或 false}}]}}"""
|
||||
|
||||
_TAXONOMY_PROMPT = """你是一个文档分类审核员。请判断下面的【摘要】所描述的文档是否适合归入类目【{category}】。
|
||||
|
||||
【摘要】
|
||||
{summary}
|
||||
|
||||
只输出 JSON，格式：{{"consistent": true 或 false}}"""
|
||||
|
||||
|
||||
def _extract_json(text: str) -> dict | None:
|
||||
"""从模型输出提取首个 JSON 对象（容忍前后噪声），失败返回 None"""
|
||||
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(match.group())
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
async def hallucination_rate(summary: str, source_text: str, ollama: OllamaClient) -> float:
|
||||
"""幻觉率：摘要中不被原文支持的断言占比
|
||||
|
||||
流程：模型抽取断言（最多 _MAX_CLAIMS 条）→ 逐条判原文是否支持 → 返回不支持比例。
|
||||
调用或解析失败保守返回 0.0 并记录日志。
|
||||
"""
|
||||
prompt = _HALLUCINATION_PROMPT.format(max_claims=_MAX_CLAIMS, source=source_text, summary=summary)
|
||||
try:
|
||||
raw = await ollama.generate(prompt, json_mode=True)
|
||||
except Exception as exc:
|
||||
logger.warning("幻觉率判定调用失败，保守返回 0.0", error=str(exc))
|
||||
return 0.0
|
||||
|
||||
data = _extract_json(raw)
|
||||
if data is None:
|
||||
logger.warning("幻觉率判定输出解析失败，保守返回 0.0", raw=raw[:200])
|
||||
return 0.0
|
||||
|
||||
claims = data.get("assertions")
|
||||
if not isinstance(claims, list):
|
||||
logger.warning("幻觉率判定输出缺少 assertions 字段，保守返回 0.0", raw=raw[:200])
|
||||
return 0.0
|
||||
supported_flags = [bool(c["supported"]) for c in claims[:_MAX_CLAIMS] if isinstance(c, dict) and "supported" in c]
|
||||
if not supported_flags:
|
||||
return 0.0
|
||||
unsupported = sum(1 for supported in supported_flags if not supported)
|
||||
return unsupported / len(supported_flags)
|
||||
|
||||
|
||||
async def taxonomy_consistency(summary: str, category: str, ollama: OllamaClient) -> bool:
|
||||
"""类目一致性：摘要是否支持归入指定类目；调用或解析失败保守返回 True"""
|
||||
prompt = _TAXONOMY_PROMPT.format(category=category, summary=summary)
|
||||
try:
|
||||
raw = await ollama.generate(prompt, json_mode=True)
|
||||
except Exception as exc:
|
||||
logger.warning("类目一致性判定调用失败，保守返回 True", error=str(exc))
|
||||
return True
|
||||
|
||||
data = _extract_json(raw)
|
||||
if data is None or "consistent" not in data:
|
||||
logger.warning("类目一致性判定输出解析失败，保守返回 True", raw=raw[:200])
|
||||
return True
|
||||
return bool(data["consistent"])
|
||||
@@ -0,0 +1,106 @@
|
||||
"""离线评测指标：纯函数实现，不依赖 Qdrant / Ollama，可独立单测
|
||||
|
||||
- entity_recall：实体保留率（摘要质量）
|
||||
- routing_f1：L1 路由层 micro P/R/F1
|
||||
- pruning_loss：上层剪枝损失率
|
||||
- precision_at_k / recall_at_k：检索效用
|
||||
- aggregate：按 key 汇总均值
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
# 数字串（含小数/版本号/百分数），如 1536、3.12、95%
|
||||
_NUMBER_RE = re.compile(r"\d+(?:\.\d+)*%?")
|
||||
# 型号/版本号模式，如 bge-m3、v1.2.0、FD-07
|
||||
_MODEL_RE = re.compile(r"[A-Za-z]+[-_]\w*\d\w*|[A-Za-z]+\d+(?:\.\d+)*")
|
||||
# 条款号，如 第三条、第 5 章、第十条
|
||||
_CLAUSE_RE = re.compile(r"第\s*[一二三四五六七八九十百千万零0-9]+\s*[条款章节项]")
|
||||
# 英文/大小写混合词（含连字符/点号连接），如 Qdrant、OpenAI、text-embedding-3-small
|
||||
_EN_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*(?:[-_.][A-Za-z0-9]+)*")
|
||||
# 中文书名号内词，如 《部署手册》
|
||||
_BOOK_TITLE_RE = re.compile(r"《([^》]+)》")
|
||||
|
||||
|
||||
def _extract_entities(text: str) -> set[str]:
|
||||
"""从文本抽取关键实体 token（去重集合）"""
|
||||
entities: set[str] = set()
|
||||
entities.update(_NUMBER_RE.findall(text))
|
||||
entities.update(_MODEL_RE.findall(text))
|
||||
entities.update(m.replace(" ", "") for m in _CLAUSE_RE.findall(text))
|
||||
entities.update(_BOOK_TITLE_RE.findall(text))
|
||||
# 英文词至少 2 个字符，避免单字母噪声
|
||||
entities.update(w for w in _EN_WORD_RE.findall(text) if len(w) >= 2)
|
||||
return entities
|
||||
|
||||
|
||||
def entity_recall(source_text: str, summary: str) -> float:
|
||||
"""实体保留率：原文抽取的关键实体在摘要中保留的比例
|
||||
|
||||
原文无可抽取实体时返回 1.0（无实体可丢，视为满分）。
|
||||
"""
|
||||
entities = _extract_entities(source_text)
|
||||
if not entities:
|
||||
return 1.0
|
||||
kept = sum(1 for entity in entities if entity in summary)
|
||||
return kept / len(entities)
|
||||
|
||||
|
||||
def routing_f1(golden_doc_ids_per_query: list[set[str]], routed_doc_ids_per_query: list[set[str]]) -> dict[str, float]:
|
||||
"""L1 路由层 micro precision / recall / f1
|
||||
|
||||
逐 query 累加 tp/fp/fn 后统一计算（micro 平均）；
|
||||
两条列表必须等长，通常只统计 positive query（negative query 无 golden doc）。
|
||||
"""
|
||||
tp = fp = fn = 0
|
||||
for golden, routed in zip(golden_doc_ids_per_query, routed_doc_ids_per_query, strict=True):
|
||||
tp += len(golden & routed)
|
||||
fp += len(routed - golden)
|
||||
fn += len(golden - routed)
|
||||
precision = tp / (tp + fp) if tp + fp else 0.0
|
||||
recall = tp / (tp + fn) if tp + fn else 0.0
|
||||
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
|
||||
return {"precision": precision, "recall": recall, "f1": f1}
|
||||
|
||||
|
||||
def pruning_loss(cases: list[bool]) -> float:
|
||||
"""剪枝损失率：原文含答案但 L1 未命中 golden doc 的 positive query 占比
|
||||
|
||||
cases 中每个元素表示一条 positive query 是否被上层剪掉（True = 被剪）。
|
||||
"""
|
||||
if not cases:
|
||||
return 0.0
|
||||
return sum(cases) / len(cases)
|
||||
|
||||
|
||||
def precision_at_k(hit_doc_ids: list[str], golden: set[str], k: int) -> float:
|
||||
"""Precision@k：前 k 条命中中 doc_id 属于 golden 的比例（按命中条数计，不去重）"""
|
||||
if k <= 0:
|
||||
return 0.0
|
||||
top = hit_doc_ids[:k]
|
||||
if not top:
|
||||
return 0.0
|
||||
hits = sum(1 for doc_id in top if doc_id in golden)
|
||||
return hits / k
|
||||
|
||||
|
||||
def recall_at_k(hit_doc_ids: list[str], golden: set[str], k: int) -> float:
|
||||
"""Recall@k：前 k 条命中覆盖的 golden doc 比例（按 doc 去重）；golden 为空返回 0.0"""
|
||||
if not golden:
|
||||
return 0.0
|
||||
found = {doc_id for doc_id in hit_doc_ids[:k] if doc_id in golden}
|
||||
return len(found) / len(golden)
|
||||
|
||||
|
||||
def aggregate(records: list[dict[str, float]]) -> dict[str, float]:
|
||||
"""按 key 汇总均值：对记录列表中每个指标求算术平均
|
||||
|
||||
某 key 只在部分记录中出现时，按出现的记录取均值；空列表返回 {}。
|
||||
"""
|
||||
if not records:
|
||||
return {}
|
||||
keys = list(dict.fromkeys(key for record in records for key in record))
|
||||
result: dict[str, float] = {}
|
||||
for key in keys:
|
||||
values = [record[key] for record in records if key in record]
|
||||
result[key] = sum(values) / len(values) if values else 0.0
|
||||
return result
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"documents": [
|
||||
{
|
||||
"id": "doc001",
|
||||
"title": "QMDSearch 向量检索服务部署运维手册",
|
||||
"text": "# 部署概述\nQMDSearch v1.2.0 依赖 Qdrant 1.12、Redis 7.2 与 Ollama 0.5 三个组件，默认端口分别为 6333、6379 与 11434。生产环境推荐使用 docker compose 一键拉起，健康检查路径为 /health，首次启动约需 30 秒完成集合初始化。\n# 关键配置\nembedding_dimension 默认 1536，sparse_enabled 默认开启。按照《运维规范》第三条要求：修改 retrieval_top_k 或 l1_doc_top_n 后必须重启服务，并执行回归冒烟用例。\n# 故障排查\n若 Qdrant 连接超时，先检查容器网络 qmd-net 是否互通；Ollama 拉取模型失败时重试 ollama pull qwen2.5:1.5b；Redis 不可用时会自动降级为直连检索，不影响主流程。",
|
||||
"golden_category": "技术文档"
|
||||
},
|
||||
{
|
||||
"id": "doc002",
|
||||
"title": "Embedding 服务接入指南",
|
||||
"text": "# 接入方式\nEmbeddingService 统一接口支持 openai 与 local 两种 provider。openai provider 默认模型 text-embedding-3-small，走 OpenAI 兼容 API；local provider 调用 Ollama 的 /api/embed 接口，默认模型 bge-m3，超时 60 秒。\n# 批量调用\nembed(texts) 支持批量编码，传入空列表时直接返回空列表。单次批量建议不超过 64 条文本，超长文本建议先按 800 字符切分。\n# 维度校验\n返回向量维度与 settings.embedding_dimension 不一致时仅记录 warning 不抛错，且每次调用最多提示一次，避免日志刷屏。",
|
||||
"golden_category": "技术文档"
|
||||
},
|
||||
{
|
||||
"id": "doc003",
|
||||
"title": "智能搜索助手 Pro 2.0 产品说明书",
|
||||
"text": "# 产品简介\n智能搜索助手 Pro 2.0 面向企业知识库场景，单机支持 200 人并发查询，平均响应时间 350 毫秒，P99 延迟不超过 1.2 秒。\n# 核心功能\n产品支持分层摘要检索、类目路由与平铺全文检索三种模式，可按租户切换。第 5 章介绍高级筛选语法，更多示例见《用户操作手册》。\n# 版本记录\n2.0 版新增 RRF 融合排序与 sparse 稀疏检索；1.8 版的旧版筛选语法兼容至 2026 年 12 月 31 日，之后停止维护。",
|
||||
"golden_category": "产品手册"
|
||||
},
|
||||
{
|
||||
"id": "doc004",
|
||||
"title": "云文档协作平台快速上手指南",
|
||||
"text": "# 快速开始\n注册账号后 3 分钟内即可创建首个知识空间，免费额度包含 5GB 存储与每月 1000 次 API 调用，超出后按 0.01 元每次计费。\n# 协作功能\n平台支持多人实时编辑、行内评论与版本回滚，历史版本默认保留 90 天，企业版可延长至 365 天。详见《协作白皮书》第二章。\n# 移动端\niOS 与 Android 客户端支持离线缓存，单文件上限 200MB，弱网环境下自动启用增量同步。",
|
||||
"golden_category": "产品手册"
|
||||
},
|
||||
{
|
||||
"id": "doc005",
|
||||
"title": "差旅费用报销管理办法",
|
||||
"text": "# 适用范围\n本办法适用于全体正式员工与实习生，自 2026 年 1 月 1 日起施行，原 2024 版办法同时废止。\n# 报销标准\n一线城市住宿限额每晚 500 元，二线城市 400 元，其他城市 300 元；高铁二等座、飞机经济舱据实报销。第十条规定：超标部分需部门总监书面审批。\n# 报销流程\n发票开具后 30 日内须提交 OA 系统，超过 90 天的票据不予受理。常见问题见《财务报销常见问题》第七条。",
|
||||
"golden_category": "财务行政"
|
||||
},
|
||||
{
|
||||
"id": "doc006",
|
||||
"title": "固定资产采购与领用规定",
|
||||
"text": "# 采购审批\n单笔金额超过 5000 元的采购须走 OA 审批流，超过 20000 元需分管副总裁签字，紧急采购可先邮件报备后补流程。\n# 领用登记\n固定资产领用后 3 个工作日内在行政系统登记资产编号，编号规则见《资产管理细则》第四条，笔记本等移动设备须加贴防伪标签。\n# 盘点与报废\n每年 12 月进行年度盘点，报废资产须填写 FD-07 表单并附照片存档，残值率按 5% 计提。",
|
||||
"golden_category": "财务行政"
|
||||
}
|
||||
],
|
||||
"queries": [
|
||||
{"query": "QMDSearch 依赖哪些组件，分别用什么端口？", "type": "positive", "golden_doc_id": "doc001", "golden_section": "部署概述"},
|
||||
{"query": "修改 retrieval_top_k 之后需要做什么？", "type": "positive", "golden_doc_id": "doc001", "golden_section": "关键配置"},
|
||||
{"query": "Ollama 拉取模型失败应该怎么处理？", "type": "positive", "golden_doc_id": "doc001", "golden_section": "故障排查"},
|
||||
{"query": "embedding_dimension 的默认值是多少？", "type": "positive", "golden_doc_id": "doc001", "golden_section": "关键配置"},
|
||||
{"query": "公司组织年度体检的医院是哪家？", "type": "negative", "golden_doc_id": null},
|
||||
|
||||
{"query": "local provider 默认使用哪个嵌入模型？", "type": "positive", "golden_doc_id": "doc002", "golden_section": "接入方式"},
|
||||
{"query": "embed 接口传入空列表会返回什么？", "type": "positive", "golden_doc_id": "doc002", "golden_section": "批量调用"},
|
||||
{"query": "向量维度和配置不一致时会抛异常吗？", "type": "positive", "golden_doc_id": "doc002", "golden_section": "维度校验"},
|
||||
{"query": "如何申请欧洲申根旅游签证？", "type": "negative", "golden_doc_id": null},
|
||||
|
||||
{"query": "智能搜索助手 Pro 2.0 支持多少并发查询？", "type": "positive", "golden_doc_id": "doc003", "golden_section": "产品简介"},
|
||||
{"query": "高级筛选语法在产品说明书的哪一章介绍？", "type": "positive", "golden_doc_id": "doc003", "golden_section": "核心功能"},
|
||||
{"query": "旧版筛选语法兼容到什么时候？", "type": "positive", "golden_doc_id": "doc003", "golden_section": "版本记录"},
|
||||
{"query": "竞品的按年订阅价格是多少？", "type": "negative", "golden_doc_id": null},
|
||||
|
||||
{"query": "云文档平台免费额度包含多少存储空间？", "type": "positive", "golden_doc_id": "doc004", "golden_section": "快速开始"},
|
||||
{"query": "历史版本默认保留多长时间？", "type": "positive", "golden_doc_id": "doc004", "golden_section": "协作功能"},
|
||||
{"query": "移动端单文件上传上限是多少？", "type": "positive", "golden_doc_id": "doc004", "golden_section": "移动端"},
|
||||
{"query": "视频会议最多支持多少人同时开启摄像头？", "type": "negative", "golden_doc_id": null},
|
||||
|
||||
{"query": "一线城市住宿报销限额是多少？", "type": "positive", "golden_doc_id": "doc005", "golden_section": "报销标准"},
|
||||
{"query": "发票超过多少天就不能报销了？", "type": "positive", "golden_doc_id": "doc005", "golden_section": "报销流程"},
|
||||
{"query": "住宿超标部分需要谁审批？", "type": "positive", "golden_doc_id": "doc005", "golden_section": "报销标准"},
|
||||
{"query": "新版差旅报销办法什么时候开始施行？", "type": "positive", "golden_doc_id": "doc005", "golden_section": "适用范围"},
|
||||
{"query": "员工结婚礼金的公司福利标准是多少？", "type": "negative", "golden_doc_id": null},
|
||||
|
||||
{"query": "采购金额超过多少需要副总裁签字？", "type": "positive", "golden_doc_id": "doc006", "golden_section": "采购审批"},
|
||||
{"query": "资产编号规则在哪个文件里规定的？", "type": "positive", "golden_doc_id": "doc006", "golden_section": "领用登记"},
|
||||
{"query": "报废资产需要填写什么表单？", "type": "positive", "golden_doc_id": "doc006", "golden_section": "盘点与报废"},
|
||||
{"query": "办公区绿植养护的排班表在哪里查？", "type": "negative", "golden_doc_id": null}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""分层摘要 RAG 离线评测 harness
|
||||
|
||||
流程：回归集 → 独立 _eval 后缀集合入库（Ingester）→ 摘要质量指标（Entity Recall /
|
||||
幻觉率 / 类目一致性）→ 检索效用指标（Routing F1 / Pruning Loss / Precision@5 /
|
||||
Recall@10）→ 平铺 chunks baseline 对比 → Markdown 报告（stdout + report.md）。
|
||||
|
||||
用法：
|
||||
uv run python scripts/eval/run_eval.py [--regression PATH] [--keep-data] [--no-judge]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 脚本直运行时把项目根加入 sys.path，保证可以 import app 与 scripts.eval
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
import structlog # noqa: E402
|
||||
|
||||
from app.config import settings # noqa: E402
|
||||
from app.core import ingestion as ingestion_mod # noqa: E402
|
||||
from app.core import retriever as retriever_mod # noqa: E402
|
||||
from app.core.ingestion import Ingester # noqa: E402
|
||||
from app.core.retriever import Retriever # noqa: E402
|
||||
from app.models.document import DocumentInput # noqa: E402
|
||||
from app.models.search import SearchRequest # noqa: E402
|
||||
from app.services import qdrant as qdrant_mod # noqa: E402
|
||||
from app.services.ollama import OllamaClient # noqa: E402
|
||||
from app.services.qdrant import QdrantService # noqa: E402
|
||||
from scripts.eval.judge import hallucination_rate, taxonomy_consistency # noqa: E402
|
||||
from scripts.eval.metrics import ( # noqa: E402
|
||||
aggregate,
|
||||
entity_recall,
|
||||
precision_at_k,
|
||||
pruning_loss,
|
||||
recall_at_k,
|
||||
routing_f1,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 评测集合后缀与默认路径
|
||||
EVAL_SUFFIX = "_eval"
|
||||
_SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_REGRESSION = _SCRIPT_DIR / "regression_set.json"
|
||||
REPORT_PATH = _SCRIPT_DIR / "report.md"
|
||||
|
||||
# 门槛（Spec 规定）：低于/高于门槛在报告中标红
|
||||
THRESHOLD_L1_ER = 0.85 # L1 Entity Recall ≥ 0.85
|
||||
THRESHOLD_L3_ER = 0.9 # L3 Entity Recall ≥ 0.9
|
||||
THRESHOLD_HALLUCINATION = 0.02 # 幻觉率 < 2%
|
||||
THRESHOLD_PRUNING_LOSS = 0.08 # Pruning Loss < 8%
|
||||
|
||||
|
||||
def _switch_to_eval_collections() -> list[str]:
|
||||
"""把 app 内模块级集合名常量整体切换为 _eval 后缀的评测集合
|
||||
|
||||
QdrantService / Retriever / Ingester 均在各自模块命名空间引用了集合名常量，
|
||||
评测脚本统一改写这些模块属性实现集合隔离，不改动 app 源码。
|
||||
返回评测集合名列表（用于评测结束后清理）。
|
||||
"""
|
||||
eval_names = {
|
||||
"COLLECTION_L1": f"{qdrant_mod.COLLECTION_L1}{EVAL_SUFFIX}",
|
||||
"COLLECTION_L2": f"{qdrant_mod.COLLECTION_L2}{EVAL_SUFFIX}",
|
||||
"COLLECTION_L3": f"{qdrant_mod.COLLECTION_L3}{EVAL_SUFFIX}",
|
||||
"COLLECTION_CHUNKS": f"{qdrant_mod.COLLECTION_CHUNKS}{EVAL_SUFFIX}",
|
||||
}
|
||||
for module in (qdrant_mod, retriever_mod, ingestion_mod):
|
||||
for name, value in eval_names.items():
|
||||
if hasattr(module, name):
|
||||
setattr(module, name, value)
|
||||
qdrant_mod.ALL_COLLECTIONS = tuple(eval_names.values())
|
||||
sparse_eval = (eval_names["COLLECTION_L1"], eval_names["COLLECTION_CHUNKS"])
|
||||
qdrant_mod.SPARSE_COLLECTIONS = sparse_eval
|
||||
retriever_mod.SPARSE_COLLECTIONS = sparse_eval
|
||||
# upsert_nodes 按集合名校验层级前缀，需同步替换
|
||||
qdrant_mod._NODE_ID_PREFIX = {eval_names["COLLECTION_L2"]: "l2", eval_names["COLLECTION_L3"]: "l3"}
|
||||
return list(eval_names.values())
|
||||
|
||||
|
||||
async def _check_services(qdrant: QdrantService, ollama: OllamaClient) -> str | None:
|
||||
"""检查 Qdrant / Ollama 连通性，返回错误消息（None 表示正常）"""
|
||||
try:
|
||||
await qdrant.client.get_collections()
|
||||
except Exception as exc:
|
||||
return f"无法连接 Qdrant（{settings.qdrant_host}:{settings.qdrant_port}）：{exc}"
|
||||
if not await ollama.is_available():
|
||||
return f"无法连接 Ollama（{settings.ollama_base_url}），请确认服务已启动（入库与评测均依赖 Ollama）"
|
||||
return None
|
||||
|
||||
|
||||
async def _l1_candidate_doc_ids(retriever: Retriever, query: str) -> set[str]:
|
||||
"""轻量复现 Retriever 的 L1 路由层：embed → L1 集合 top-N → 候选 doc_id 集合
|
||||
|
||||
用于计算 Pruning Loss 与 Routing F1；不套类目过滤，度量纯 L1 向量召回。
|
||||
"""
|
||||
dense = (await retriever.embedding.embed([query]))[0]
|
||||
sparse = retriever.sparse_encoder.encode(query) if settings.sparse_enabled else None
|
||||
# 复用 Retriever 内部检索方法（hybrid/dense 按集合能力自动选择）
|
||||
hits = await retriever._search_collection(retriever_mod.COLLECTION_L1, dense, sparse, settings.l1_doc_top_n, None)
|
||||
return {(p.payload or {}).get("doc_id", "") for p in hits} - {""}
|
||||
|
||||
|
||||
async def _baseline_doc_ids(retriever: Retriever, query: str) -> list[str]:
|
||||
"""平铺 baseline：直接对 chunks 集合做 hybrid top-k（无路由无剪枝），返回命中 doc_id 列表"""
|
||||
dense = (await retriever.embedding.embed([query]))[0]
|
||||
sparse = retriever.sparse_encoder.encode(query) if settings.sparse_enabled else None
|
||||
points = await retriever._search_collection(
|
||||
retriever_mod.COLLECTION_CHUNKS, dense, sparse, settings.retrieval_top_k, None
|
||||
)
|
||||
return [(p.payload or {}).get("doc_id", "") for p in points]
|
||||
|
||||
|
||||
def _mark(ok: bool) -> str:
|
||||
"""门槛判定标记"""
|
||||
return "✅" if ok else "❌"
|
||||
|
||||
|
||||
def _fmt(value: float | None, percent: bool = False) -> str:
|
||||
"""数值格式化；None 表示未评测（judge 跳过）"""
|
||||
if value is None:
|
||||
return "N/A"
|
||||
return f"{value:.1%}" if percent else f"{value:.4f}"
|
||||
|
||||
|
||||
def _build_report(
|
||||
regression_path: Path,
|
||||
doc_records: list[dict],
|
||||
query_summary: dict,
|
||||
hier_metrics: dict[str, float],
|
||||
baseline_metrics: dict[str, float],
|
||||
routing: dict[str, float],
|
||||
prune_loss: float,
|
||||
judge_enabled: bool,
|
||||
kept_data: bool,
|
||||
) -> str:
|
||||
"""组装 Markdown 评测报告"""
|
||||
lines: list[str] = [
|
||||
"# 分层摘要 RAG 评测报告",
|
||||
"",
|
||||
f"- 回归集：`{regression_path}`",
|
||||
f"- 文档数：{len(doc_records)}；query 数：{query_summary['total']}"
|
||||
f"（positive {query_summary['positive']} / negative {query_summary['negative']}）",
|
||||
f"- LLM judge：{'开启' if judge_enabled else '跳过（--no-judge 或 Ollama 不可用）'}",
|
||||
f"- 评测集合：`*{EVAL_SUFFIX}`（{'保留' if kept_data else '已清理'}）",
|
||||
"",
|
||||
"## 摘要质量（按文档）",
|
||||
"",
|
||||
"| 文档 | 标题 | golden 类目 | 实际类目 | L1 Entity Recall | L3 Entity Recall | 幻觉率 | 类目一致 |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for record in doc_records:
|
||||
consistency = record.get("taxonomy_consistency")
|
||||
lines.append(
|
||||
f"| {record['id']} | {record['title']} | {record['golden_category']} | {record['category']} "
|
||||
f"| {_fmt(record['entity_recall_l1'])} | {_fmt(record['entity_recall_l3'])} "
|
||||
f"| {_fmt(record.get('hallucination_rate'), percent=True)} "
|
||||
f"| {('是' if consistency else '否') if consistency is not None else 'N/A'} |"
|
||||
)
|
||||
|
||||
# 汇总与门槛判定
|
||||
er_l1 = aggregate([{"v": r["entity_recall_l1"]} for r in doc_records]).get("v", 0.0)
|
||||
er_l3 = aggregate([{"v": r["entity_recall_l3"]} for r in doc_records]).get("v", 0.0)
|
||||
hall_records = [{"v": r["hallucination_rate"]} for r in doc_records if r.get("hallucination_rate") is not None]
|
||||
hall = aggregate(hall_records).get("v") if hall_records else None
|
||||
lines += [
|
||||
"",
|
||||
"## 指标汇总与门槛判定",
|
||||
"",
|
||||
"| 指标 | 数值 | 门槛 | 判定 |",
|
||||
"| --- | --- | --- | --- |",
|
||||
f"| L1 Entity Recall | {_fmt(er_l1)} | ≥ {THRESHOLD_L1_ER} | {_mark(er_l1 >= THRESHOLD_L1_ER)} |",
|
||||
f"| L3 Entity Recall | {_fmt(er_l3)} | ≥ {THRESHOLD_L3_ER} | {_mark(er_l3 >= THRESHOLD_L3_ER)} |",
|
||||
f"| Hallucination Rate | {_fmt(hall, percent=True)} | < {THRESHOLD_HALLUCINATION:.0%} "
|
||||
f"| {_mark(hall < THRESHOLD_HALLUCINATION) if hall is not None else 'N/A'} |",
|
||||
f"| Pruning Loss | {_fmt(prune_loss, percent=True)} | < {THRESHOLD_PRUNING_LOSS:.0%} "
|
||||
f"| {_mark(prune_loss < THRESHOLD_PRUNING_LOSS)} |",
|
||||
"",
|
||||
"## 检索效用（hierarchical vs 平铺 baseline）",
|
||||
"",
|
||||
"| 指标 | 分层检索 | 平铺 baseline |",
|
||||
"| --- | --- | --- |",
|
||||
f"| Precision@5 | {_fmt(hier_metrics.get('precision@5', 0.0))} "
|
||||
f"| {_fmt(baseline_metrics.get('precision@5', 0.0))} |",
|
||||
f"| Recall@10 | {_fmt(hier_metrics.get('recall@10', 0.0))} | {_fmt(baseline_metrics.get('recall@10', 0.0))} |",
|
||||
"",
|
||||
"### 路由层（L1）",
|
||||
"",
|
||||
f"- Routing Precision：{_fmt(routing['precision'])}",
|
||||
f"- Routing Recall：{_fmt(routing['recall'])}",
|
||||
f"- Routing F1：{_fmt(routing['f1'])}",
|
||||
f"- Pruning Loss：{_fmt(prune_loss, percent=True)}（{_mark(prune_loss < THRESHOLD_PRUNING_LOSS)}）",
|
||||
"",
|
||||
"### negative query",
|
||||
"",
|
||||
f"- 误中（返回了任意结果）：{query_summary['negative_false_alarm']} / {query_summary['negative']}",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def run(regression_path: Path, keep_data: bool, judge_enabled: bool) -> int:
|
||||
"""评测主流程，返回进程退出码"""
|
||||
data = json.loads(regression_path.read_text(encoding="utf-8"))
|
||||
documents: list[dict] = data["documents"]
|
||||
queries: list[dict] = data["queries"]
|
||||
|
||||
eval_collections = _switch_to_eval_collections()
|
||||
qdrant = QdrantService()
|
||||
ollama = OllamaClient()
|
||||
|
||||
# 连通性检查：失败给出中文提示并以退出码 2 结束
|
||||
error = await _check_services(qdrant, ollama)
|
||||
if error:
|
||||
print(f"错误：{error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
await qdrant.ensure_collections()
|
||||
logger.info("评测集合就绪", collections=eval_collections)
|
||||
|
||||
ingester = Ingester(qdrant=qdrant)
|
||||
retriever = Retriever(qdrant=qdrant)
|
||||
|
||||
id_map: dict[str, str] = {} # 回归集 doc id -> 实际入库 doc_id
|
||||
doc_records: list[dict] = []
|
||||
hier_records: list[dict[str, float]] = []
|
||||
baseline_records: list[dict[str, float]] = []
|
||||
pruned_cases: list[bool] = []
|
||||
golden_sets: list[set[str]] = []
|
||||
routed_sets: list[set[str]] = []
|
||||
negative_total = 0
|
||||
negative_false_alarm = 0
|
||||
|
||||
try:
|
||||
# 1. 逐篇入库并计算摘要质量指标
|
||||
for doc in documents:
|
||||
result = await ingester.ingest(DocumentInput(title=doc["title"], text=doc["text"]))
|
||||
id_map[doc["id"]] = result.document_id
|
||||
logger.info("文档入库完成", id=doc["id"], doc_id=result.document_id, category=result.category)
|
||||
|
||||
record: dict = {
|
||||
"id": doc["id"],
|
||||
"title": doc["title"],
|
||||
"golden_category": doc["golden_category"],
|
||||
"category": result.category,
|
||||
"entity_recall_l1": entity_recall(doc["text"], result.summary.l1_summary),
|
||||
"entity_recall_l3": entity_recall(doc["text"], result.summary.l3_content_outline),
|
||||
"hallucination_rate": None,
|
||||
"taxonomy_consistency": None,
|
||||
}
|
||||
if judge_enabled:
|
||||
record["hallucination_rate"] = await hallucination_rate(result.summary.l1_summary, doc["text"], ollama)
|
||||
record["taxonomy_consistency"] = await taxonomy_consistency(
|
||||
result.summary.l1_summary, result.category, ollama
|
||||
)
|
||||
doc_records.append(record)
|
||||
|
||||
# 2. 逐 query 计算检索效用指标（分层检索 + 轻量 L1 路由层 + 平铺 baseline）
|
||||
for query in queries:
|
||||
query_text = query["query"]
|
||||
golden_ids = {id_map[query["golden_doc_id"]]} if query.get("golden_doc_id") else set()
|
||||
|
||||
response = await retriever.search(SearchRequest(query=query_text))
|
||||
hit_doc_ids = [hit.doc_id for hit in response.hits]
|
||||
l1_candidates = await _l1_candidate_doc_ids(retriever, query_text)
|
||||
|
||||
if query["type"] == "positive" and golden_ids:
|
||||
pruned_cases.append(not golden_ids & l1_candidates)
|
||||
golden_sets.append(golden_ids)
|
||||
routed_sets.append(l1_candidates)
|
||||
hier_records.append(
|
||||
{
|
||||
"precision@5": precision_at_k(hit_doc_ids, golden_ids, 5),
|
||||
"recall@10": recall_at_k(hit_doc_ids, golden_ids, 10),
|
||||
}
|
||||
)
|
||||
baseline_ids = await _baseline_doc_ids(retriever, query_text)
|
||||
baseline_records.append(
|
||||
{
|
||||
"precision@5": precision_at_k(baseline_ids, golden_ids, 5),
|
||||
"recall@10": recall_at_k(baseline_ids, golden_ids, 10),
|
||||
}
|
||||
)
|
||||
else:
|
||||
# negative query：期望无结果，任何返回均计为误中
|
||||
negative_total += 1
|
||||
if hit_doc_ids:
|
||||
negative_false_alarm += 1
|
||||
finally:
|
||||
# 评测集合清理（--keep-data 时保留）
|
||||
if keep_data:
|
||||
logger.info("--keep-data 生效，保留评测集合", collections=eval_collections)
|
||||
else:
|
||||
for collection in eval_collections:
|
||||
try:
|
||||
await qdrant.client.delete_collection(collection)
|
||||
except Exception as exc:
|
||||
logger.warning("评测集合清理失败", collection=collection, error=str(exc))
|
||||
logger.info("评测集合已清理", collections=eval_collections)
|
||||
|
||||
# 3. 汇总并输出报告
|
||||
prune_loss = pruning_loss(pruned_cases)
|
||||
routing = routing_f1(golden_sets, routed_sets)
|
||||
hier_metrics = aggregate(hier_records)
|
||||
baseline_metrics = aggregate(baseline_records)
|
||||
query_summary = {
|
||||
"total": len(queries),
|
||||
"positive": len(queries) - negative_total,
|
||||
"negative": negative_total,
|
||||
"negative_false_alarm": negative_false_alarm,
|
||||
}
|
||||
report = _build_report(
|
||||
regression_path,
|
||||
doc_records,
|
||||
query_summary,
|
||||
hier_metrics,
|
||||
baseline_metrics,
|
||||
routing,
|
||||
prune_loss,
|
||||
judge_enabled,
|
||||
keep_data,
|
||||
)
|
||||
print(report)
|
||||
REPORT_PATH.write_text(report + "\n", encoding="utf-8")
|
||||
logger.info("评测报告已写入", path=str(REPORT_PATH))
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="分层摘要 RAG 离线评测：回归集入库 → 摘要质量/检索效用指标 → 平铺 baseline 对比 → Markdown 报告",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--regression",
|
||||
type=Path,
|
||||
default=DEFAULT_REGRESSION,
|
||||
help=f"回归集 JSON 路径（默认 {DEFAULT_REGRESSION}）",
|
||||
)
|
||||
parser.add_argument("--keep-data", action="store_true", help="评测结束后保留 _eval 集合（默认删除）")
|
||||
parser.add_argument("--no-judge", action="store_true", help="跳过 LLM-as-judge 指标（幻觉率 / 类目一致性）")
|
||||
args = parser.parse_args()
|
||||
return asyncio.run(run(args.regression.resolve(), args.keep_data, not args.no_judge))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user