Initial commit: QMDSearch 分层信息检索服务
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
"""文档 chunk 切分器
|
||||
|
||||
按文档原生标题树切分 chunk：
|
||||
- 结构化文本：每个标题起点切分 section（标题行到下一个标题前），
|
||||
超长 section 按空行段落二次切分，单段落仍超长则硬切
|
||||
- 无结构文本：直接按空行段落累加切分
|
||||
|
||||
每个 chunk 记录 section_path（祖先标题链，" / " 连接），与 L2 大纲节点互相定位。
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.core.headings import Heading, parse_headings
|
||||
from app.models.document import ChunkModel
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 段落分隔：一个或多个空行
|
||||
_PARAGRAPH_SPLIT_PATTERN = re.compile(r"\n\s*\n")
|
||||
|
||||
|
||||
class Chunker:
|
||||
"""按标题树切分文档 chunk"""
|
||||
|
||||
def __init__(self, max_chars: int = settings.chunk_max_chars) -> None:
|
||||
self.max_chars = max_chars
|
||||
|
||||
def chunk(self, text: str, doc_id: str) -> list[ChunkModel]:
|
||||
"""将文档文本切分为 chunk 列表
|
||||
|
||||
Args:
|
||||
text: 文档纯文本内容
|
||||
doc_id: 文档 ID
|
||||
|
||||
Returns:
|
||||
list[ChunkModel]: 切分结果，chunk_index 从 0 递增
|
||||
"""
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return []
|
||||
|
||||
# 全文不超长：整篇单 chunk
|
||||
if len(stripped) <= self.max_chars:
|
||||
return [ChunkModel(doc_id=doc_id, chunk_index=0, text=stripped)]
|
||||
|
||||
headings = parse_headings(stripped)
|
||||
chunks: list[ChunkModel] = []
|
||||
|
||||
if headings:
|
||||
# 结构化：先按标题切分 section，再按长度二次切分
|
||||
for section_text, section_path in self._split_sections(stripped, headings):
|
||||
for piece in self._split_by_length(section_text):
|
||||
chunks.append(
|
||||
ChunkModel(doc_id=doc_id, chunk_index=len(chunks), text=piece, section_path=section_path)
|
||||
)
|
||||
else:
|
||||
# 无结构：直接按段落累加切分
|
||||
for piece in self._split_by_length(stripped):
|
||||
chunks.append(ChunkModel(doc_id=doc_id, chunk_index=len(chunks), text=piece))
|
||||
|
||||
logger.info("文档切分完成", doc_id=doc_id, chunks_count=len(chunks), has_headings=bool(headings))
|
||||
return chunks
|
||||
|
||||
def _split_sections(self, text: str, headings: list[Heading]) -> list[tuple[str, str]]:
|
||||
"""按标题树切分 section，返回 (section 文本, section_path) 列表
|
||||
|
||||
每个标题起点切分一个 section，section 文本含标题行本身；
|
||||
section_path 为祖先标题链（含自身标题），用 " / " 连接；
|
||||
首个标题前的引导正文归入无前缀 section（section_path 为空）。
|
||||
"""
|
||||
lines = text.splitlines()
|
||||
sections: list[tuple[str, str]] = []
|
||||
|
||||
# 首个标题前的引导内容
|
||||
preamble = "\n".join(lines[: headings[0].line_index]).strip()
|
||||
if preamble:
|
||||
sections.append((preamble, ""))
|
||||
|
||||
# 维护祖先标题栈：遇到同级或更高级标题时弹栈
|
||||
stack: list[Heading] = []
|
||||
for i, heading in enumerate(headings):
|
||||
while stack and stack[-1].level >= heading.level:
|
||||
stack.pop()
|
||||
stack.append(heading)
|
||||
|
||||
end = headings[i + 1].line_index if i + 1 < len(headings) else len(lines)
|
||||
section_text = "\n".join(lines[heading.line_index : end]).strip()
|
||||
section_path = " / ".join(h.title for h in stack)
|
||||
sections.append((section_text, section_path))
|
||||
|
||||
return sections
|
||||
|
||||
def _split_by_length(self, text: str) -> list[str]:
|
||||
"""按 max_chars 切分文本：先按空行段落累加，单段落超长则硬切"""
|
||||
if len(text) <= self.max_chars:
|
||||
return [text]
|
||||
|
||||
pieces: list[str] = []
|
||||
current = ""
|
||||
for paragraph in _PARAGRAPH_SPLIT_PATTERN.split(text):
|
||||
paragraph = paragraph.strip()
|
||||
if not paragraph:
|
||||
continue
|
||||
candidate = f"{current}\n\n{paragraph}" if current else paragraph
|
||||
if len(candidate) <= self.max_chars:
|
||||
current = candidate
|
||||
continue
|
||||
if current:
|
||||
pieces.append(current)
|
||||
current = ""
|
||||
# 单段落仍超长：按 max_chars 硬切
|
||||
if len(paragraph) > self.max_chars:
|
||||
pieces.extend(paragraph[i : i + self.max_chars] for i in range(0, len(paragraph), self.max_chars))
|
||||
else:
|
||||
current = paragraph
|
||||
if current:
|
||||
pieces.append(current)
|
||||
return pieces
|
||||
@@ -0,0 +1,139 @@
|
||||
"""文档分类器
|
||||
|
||||
入库链路第二步：基于 L1 总结，用 Ollama 小模型将文档判定为 taxonomy 中的
|
||||
主类目 + 附加标签：
|
||||
- LLM 输出解析失败 / 类目名不在 taxonomy → 归 uncategorized（confidence=0.0）
|
||||
- 置信度低于阈值 → 主类目归 uncategorized，候选类目名保留进 tags（软召回用）
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.models.knowledge import UNCATEGORIZED, CategoryResult, TaxonomyCategory, load_taxonomy
|
||||
from app.services.ollama import OllamaClient
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 从 LLM 输出中提取第一个 {...} JSON 块（贪婪匹配到最后的 }，兼容嵌套对象）
|
||||
_JSON_BLOCK_RE = re.compile(r"\{.*\}", re.DOTALL)
|
||||
|
||||
|
||||
def _extract_json(raw: str) -> dict | None:
|
||||
"""从 LLM 输出中提取 JSON 对象
|
||||
|
||||
先尝试直接解析；失败则用正则提取第一个 {...} 块再解析。
|
||||
返回 None 表示无法提取出合法的 JSON 对象。
|
||||
"""
|
||||
text = raw.strip()
|
||||
try:
|
||||
data = json.loads(text)
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
match = _JSON_BLOCK_RE.search(text)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
class Classifier:
|
||||
"""文档分类器：将 L1 总结判定为 taxonomy 主类目 + 附加标签"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
|
||||
# 合法类目名集合（含 uncategorized）
|
||||
self._valid_names = {c.name for c in self.taxonomy}
|
||||
|
||||
async def classify(self, l1_summary: str, title: str = "") -> CategoryResult:
|
||||
"""对文档进行分类判定
|
||||
|
||||
Args:
|
||||
l1_summary: 文档 L1 总结
|
||||
title: 文档标题（可选，辅助判定）
|
||||
|
||||
Returns:
|
||||
CategoryResult: 主类目 / 附加标签 / 置信度
|
||||
"""
|
||||
prompt = self._build_prompt(l1_summary, title)
|
||||
raw = await self.ollama.generate(prompt, json_mode=True)
|
||||
|
||||
data = _extract_json(raw)
|
||||
if data is None:
|
||||
logger.warning("分类失败：LLM 输出非合法 JSON", title=title, output=raw[:200])
|
||||
return CategoryResult(main_category=UNCATEGORIZED, tags=[], confidence=0.0)
|
||||
|
||||
main_category = data.get("main_category")
|
||||
confidence = data.get("confidence")
|
||||
if (
|
||||
not isinstance(main_category, str)
|
||||
or main_category not in self._valid_names
|
||||
or not isinstance(confidence, (int, float))
|
||||
or not 0 <= confidence <= 1
|
||||
):
|
||||
logger.warning(
|
||||
"分类失败：类目名不在 taxonomy 或 confidence 非法",
|
||||
title=title,
|
||||
main_category=main_category,
|
||||
confidence=confidence,
|
||||
)
|
||||
return CategoryResult(main_category=UNCATEGORIZED, tags=[], confidence=0.0)
|
||||
|
||||
tags = self._clean_tags(data.get("tags"), main_category)
|
||||
confidence = float(confidence)
|
||||
|
||||
# 低置信度软召回：主类目归 uncategorized，候选类目名保留进 tags
|
||||
if confidence < settings.classify_confidence_threshold:
|
||||
logger.info(
|
||||
"分类置信度低于阈值，归入 uncategorized",
|
||||
title=title,
|
||||
candidate=main_category,
|
||||
confidence=confidence,
|
||||
threshold=settings.classify_confidence_threshold,
|
||||
)
|
||||
if main_category != UNCATEGORIZED and main_category not in tags:
|
||||
tags.insert(0, main_category)
|
||||
return CategoryResult(main_category=UNCATEGORIZED, tags=tags, confidence=confidence)
|
||||
|
||||
return CategoryResult(main_category=main_category, tags=tags, confidence=confidence)
|
||||
|
||||
def _build_prompt(self, l1_summary: str, title: str) -> str:
|
||||
"""构造分类 prompt：列出全部 taxonomy 类目（含 uncategorized 及其用途说明）"""
|
||||
category_lines = []
|
||||
for c in self.taxonomy:
|
||||
if c.name == UNCATEGORIZED:
|
||||
category_lines.append(f"- {c.name}: 当文档跨多个类目或无法明确归入其他类目时选择此类目")
|
||||
else:
|
||||
category_lines.append(f"- {c.name}: {c.description}")
|
||||
category_block = "\n".join(category_lines)
|
||||
|
||||
prompt = (
|
||||
"你是知识库分类助手。请根据文档标题和总结，判断文档最适合归入以下哪个类目。\n\n"
|
||||
f"可选类目：\n{category_block}\n\n"
|
||||
"要求：\n"
|
||||
"- main_category 只能从上面的类目名中选择，不要输出其他名称；\n"
|
||||
"- tags 为 0~3 个附加标签（词或短语），且不能包含 main_category 本身；\n"
|
||||
"- confidence 为 0~1 之间的小数，表示对主类目判断的置信度；\n"
|
||||
"- 只输出 JSON，不要输出任何其他内容。\n\n"
|
||||
'输出格式：{"main_category": "类目名", "tags": ["标签"], "confidence": 0.0}\n\n'
|
||||
)
|
||||
if title:
|
||||
prompt += f"文档标题：{title}\n"
|
||||
prompt += f"文档总结：{l1_summary}"
|
||||
return prompt
|
||||
|
||||
@staticmethod
|
||||
def _clean_tags(raw_tags: object, main_category: str) -> list[str]:
|
||||
"""清洗 LLM 输出的 tags：仅保留非空字符串、剔除主类目名、最多 3 个"""
|
||||
if not isinstance(raw_tags, list):
|
||||
return []
|
||||
tags = [t for t in raw_tags if isinstance(t, str) and t and t != main_category]
|
||||
return tags[:3]
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Dense 向量嵌入服务
|
||||
|
||||
提供统一的 EmbeddingService 接口，支持两种 provider：
|
||||
- openai：OpenAI 兼容 API（AsyncOpenAI）
|
||||
- local：本地 Ollama /api/embed 接口
|
||||
"""
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import httpx
|
||||
import structlog
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EmbeddingService(Protocol):
|
||||
"""统一嵌入服务接口"""
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
"""批量生成文本向量
|
||||
|
||||
Args:
|
||||
texts: 待嵌入文本列表，空列表时直接返回空列表
|
||||
|
||||
Returns:
|
||||
与输入等长的向量列表
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def _check_dimension(vectors: list[list[float]], provider: str) -> None:
|
||||
"""返回维度与配置不一致时告警（不抛错），每次调用最多提示一次"""
|
||||
for vec in vectors:
|
||||
if len(vec) != settings.embedding_dimension:
|
||||
logger.warning(
|
||||
"嵌入向量维度与配置不一致",
|
||||
provider=provider,
|
||||
actual=len(vec),
|
||||
expected=settings.embedding_dimension,
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
class OpenAIEmbeddingService:
|
||||
"""OpenAI 兼容 API 嵌入服务"""
|
||||
|
||||
def __init__(self, api_key: str, base_url: str, model: str) -> None:
|
||||
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
|
||||
self._model = model
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
resp = await self._client.embeddings.create(model=self._model, input=texts)
|
||||
vectors = [item.embedding for item in resp.data]
|
||||
_check_dimension(vectors, provider="openai")
|
||||
logger.debug("OpenAI 嵌入完成", model=self._model, count=len(vectors))
|
||||
return vectors
|
||||
|
||||
|
||||
class LocalEmbeddingService:
|
||||
"""本地 Ollama 嵌入服务（/api/embed）"""
|
||||
|
||||
def __init__(self, base_url: str, model: str, timeout: float = 60.0) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.model = model
|
||||
self.timeout = timeout
|
||||
|
||||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
url = f"{self.base_url}/api/embed"
|
||||
payload = {"model": self.model, "input": texts}
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
resp = await client.post(url, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
vectors: list[list[float]] = data.get("embeddings", [])
|
||||
_check_dimension(vectors, provider="local")
|
||||
logger.debug("Ollama 嵌入完成", model=self.model, count=len(vectors))
|
||||
return vectors
|
||||
|
||||
|
||||
def create_embedding_service() -> EmbeddingService:
|
||||
"""按 settings.embedding_provider 创建嵌入服务实例"""
|
||||
if settings.embedding_provider == "local":
|
||||
return LocalEmbeddingService(
|
||||
base_url=settings.ollama_base_url,
|
||||
model=settings.ollama_embedding_model,
|
||||
)
|
||||
return OpenAIEmbeddingService(
|
||||
api_key=settings.openai_api_key,
|
||||
base_url=settings.openai_base_url,
|
||||
model=settings.embedding_model,
|
||||
)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""标题树解析器
|
||||
|
||||
从纯文本中解析文档原生标题结构，支持两类模式：
|
||||
- Markdown ATX 标题（# ~ ######，# 数量即层级）
|
||||
- 中文编号标题（第X章/节/篇、一、1.1 等编号，行长度不超过 60 字符）
|
||||
|
||||
解析结果用于 L2 大纲生成与 chunk 的 section 切分。
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class Heading(BaseModel):
|
||||
"""文档标题节点"""
|
||||
|
||||
title: str = Field(description="标题文本")
|
||||
level: int = Field(description="标题层级，1 为最顶层")
|
||||
line_index: int = Field(description="标题所在行号（从 0 开始）")
|
||||
|
||||
|
||||
# Markdown ATX 标题：1~6 个 # 后跟空白
|
||||
_ATX_PATTERN = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
|
||||
|
||||
# 中文篇章节编号：第X章/第X节/第X篇（章/篇=1 级，节=2 级）
|
||||
_CN_CHAPTER_PATTERN = re.compile(r"^第[一二三四五六七八九十百\d]+([章节篇])")
|
||||
|
||||
# 中文序号：一、二、……，固定 1 级
|
||||
_CN_ENUM_PATTERN = re.compile(r"^[一二三四五六七八九十]+、")
|
||||
|
||||
# 数字编号：1. / 1、/ 1.1 / 1.1.1 等，按点分段数定层级
|
||||
_NUM_PATTERN = re.compile(r"^(\d+(?:\.\d+)*)[、.\s]")
|
||||
|
||||
# 编号类标题行的最大长度，超过则视为正文
|
||||
_MAX_HEADING_LINE_LENGTH = 60
|
||||
|
||||
# 第X[章节篇] 后缀对应的层级
|
||||
_CN_CHAPTER_LEVELS = {"章": 1, "节": 2, "篇": 1}
|
||||
|
||||
|
||||
def parse_headings(text: str) -> list[Heading]:
|
||||
"""解析文本中的标题，按行号升序返回
|
||||
|
||||
Args:
|
||||
text: 文档纯文本内容
|
||||
|
||||
Returns:
|
||||
list[Heading]: 标题列表，无标题时返回空列表
|
||||
"""
|
||||
headings: list[Heading] = []
|
||||
for line_index, line in enumerate(text.splitlines()):
|
||||
heading = _match_heading(line, line_index)
|
||||
if heading is not None:
|
||||
headings.append(heading)
|
||||
return headings
|
||||
|
||||
|
||||
def render_outline(headings: list[Heading]) -> str:
|
||||
"""将标题树渲染为大纲文本，每行一个节点，按层级缩进"""
|
||||
return "\n".join(f"{' ' * (h.level - 1)}- {h.title}" for h in headings)
|
||||
|
||||
|
||||
def _match_heading(line: str, line_index: int) -> Heading | None:
|
||||
"""匹配单行是否为标题，是则返回 Heading，否则返回 None"""
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
|
||||
# Markdown ATX 标题（无长度限制）
|
||||
match = _ATX_PATTERN.match(stripped)
|
||||
if match:
|
||||
return Heading(title=match.group(2), level=len(match.group(1)), line_index=line_index)
|
||||
|
||||
# 编号类标题有行长度限制，过长视为正文
|
||||
if len(stripped) > _MAX_HEADING_LINE_LENGTH:
|
||||
return None
|
||||
|
||||
# 第X章/节/篇
|
||||
match = _CN_CHAPTER_PATTERN.match(stripped)
|
||||
if match:
|
||||
return Heading(title=stripped, level=_CN_CHAPTER_LEVELS[match.group(1)], line_index=line_index)
|
||||
|
||||
# 一、二、……
|
||||
if _CN_ENUM_PATTERN.match(stripped):
|
||||
return Heading(title=stripped, level=1, line_index=line_index)
|
||||
|
||||
# 数字编号，层级 = 点分段数（1.=1、1.1=2、1.1.1=3）
|
||||
match = _NUM_PATTERN.match(stripped)
|
||||
if match:
|
||||
level = match.group(1).count(".") + 1
|
||||
return Heading(title=stripped, level=level, line_index=line_index)
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,172 @@
|
||||
"""入库异步任务管理器
|
||||
|
||||
将文档入库包装为后台异步任务：submit 登记任务并立即返回 task_id，
|
||||
后台受并发上限控制执行 Ingester.ingest，并按阶段推进任务状态。
|
||||
|
||||
内存注册表为主（记录 status/created_at/updated_at/result/error），
|
||||
Redis 为持久镜像（key: ingest_task:{task_id}），每次状态迁移同步写入；
|
||||
Redis 不可用或写入失败仅记录 warning，不影响任务执行。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from app.config import Settings
|
||||
from app.core.ingestion import Ingester, IngestionError
|
||||
from app.models.document import DocumentInput
|
||||
from app.services.redis import RedisCache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# Redis 任务状态 key 前缀
|
||||
REDIS_KEY_PREFIX = "ingest_task:"
|
||||
|
||||
|
||||
class IngestTaskStatus(StrEnum):
|
||||
"""入库任务状态"""
|
||||
|
||||
PENDING = "pending" # 已登记，排队等待执行
|
||||
SUMMARIZING = "summarizing" # 三级总结中
|
||||
CLASSIFYING = "classifying" # 分类判定中
|
||||
EMBEDDING = "embedding" # 向量化中
|
||||
WRITING = "writing" # 写入 Qdrant 中
|
||||
DONE = "done" # 入库完成
|
||||
FAILED = "failed" # 入库失败
|
||||
|
||||
|
||||
# 终态集合
|
||||
TERMINAL_STATUSES: frozenset[str] = frozenset({IngestTaskStatus.DONE, IngestTaskStatus.FAILED})
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
"""当前 UTC 时间的 ISO8601 字符串"""
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class IngestTaskManager:
|
||||
"""入库异步任务管理器：登记、后台执行、状态查询与 Redis 持久镜像"""
|
||||
|
||||
def __init__(self, ingester: Ingester, redis: RedisCache | None, settings: Settings) -> None:
|
||||
self._ingester = ingester
|
||||
self._redis = redis
|
||||
self._settings = settings
|
||||
self._tasks: dict[str, dict[str, Any]] = {}
|
||||
self._semaphore = asyncio.Semaphore(settings.ingest_max_concurrency)
|
||||
# 持有后台任务与镜像任务引用，避免被 GC 提前回收
|
||||
self._background_tasks: set[asyncio.Task[None]] = set()
|
||||
self._mirror_tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
async def submit(self, doc: DocumentInput) -> str:
|
||||
"""登记入库任务并后台执行，立即返回 task_id"""
|
||||
task_id = uuid.uuid4().hex
|
||||
now = _utc_now_iso()
|
||||
self._tasks[task_id] = {
|
||||
"task_id": task_id,
|
||||
"status": IngestTaskStatus.PENDING,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"result": None,
|
||||
"error": None,
|
||||
}
|
||||
self._schedule_mirror(task_id)
|
||||
background = asyncio.create_task(self._run(task_id, doc))
|
||||
self._background_tasks.add(background)
|
||||
background.add_done_callback(self._background_tasks.discard)
|
||||
logger.info("入库任务已登记", task_id=task_id, title=doc.title)
|
||||
return task_id
|
||||
|
||||
async def get(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""查询任务状态：先查内存注册表，miss 再查 Redis 镜像，都没有返回 None"""
|
||||
record = self._tasks.get(task_id)
|
||||
if record is not None:
|
||||
return record
|
||||
if self._redis is None:
|
||||
return None
|
||||
try:
|
||||
return await self._redis.get_json(f"{REDIS_KEY_PREFIX}{task_id}")
|
||||
except Exception:
|
||||
logger.warning("入库任务状态读取 Redis 失败，降级为未命中", task_id=task_id, exc_info=True)
|
||||
return None
|
||||
|
||||
async def wait_done(self, task_id: str, timeout: float = 30.0) -> dict[str, Any]:
|
||||
"""轮询内存注册表直到任务进入终态（done/failed）或超时
|
||||
|
||||
进入终态后会等待已调度的 Redis 镜像写完再返回；超时抛 TimeoutError。
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
while True:
|
||||
record = self._tasks.get(task_id)
|
||||
if record is not None and record["status"] in TERMINAL_STATUSES:
|
||||
if self._mirror_tasks:
|
||||
await asyncio.gather(*self._mirror_tasks, return_exceptions=True)
|
||||
return record
|
||||
if loop.time() >= deadline:
|
||||
raise TimeoutError(f"入库任务 {task_id} 在 {timeout}s 内未进入终态")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def _run(self, task_id: str, doc: DocumentInput) -> None:
|
||||
"""后台执行入库：并发限流 + 阶段状态推进 + 结果/错误落账"""
|
||||
async with self._semaphore:
|
||||
try:
|
||||
result = await self._ingester.ingest(doc, progress_cb=lambda stage: self._on_progress(task_id, stage))
|
||||
except IngestionError as exc:
|
||||
# 入库已知失败：透传阶段与已产出的部分总结
|
||||
self._finish_failed(
|
||||
task_id,
|
||||
{
|
||||
"stage": exc.stage,
|
||||
"message": str(exc),
|
||||
"partial_summary": exc.summary.model_dump(mode="json") if exc.summary is not None else None,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
self._finish_failed(task_id, {"stage": "unknown", "message": str(exc), "partial_summary": None})
|
||||
else:
|
||||
self._tasks[task_id].update(
|
||||
status=IngestTaskStatus.DONE,
|
||||
updated_at=_utc_now_iso(),
|
||||
result=result.model_dump(mode="json"),
|
||||
)
|
||||
self._schedule_mirror(task_id)
|
||||
logger.info("入库任务完成", task_id=task_id)
|
||||
|
||||
def _finish_failed(self, task_id: str, error: dict[str, Any]) -> None:
|
||||
"""将任务置为 failed 并记录错误信息"""
|
||||
self._tasks[task_id].update(status=IngestTaskStatus.FAILED, updated_at=_utc_now_iso(), error=error)
|
||||
self._schedule_mirror(task_id)
|
||||
logger.error("入库任务失败", task_id=task_id, stage=error["stage"], error=error["message"])
|
||||
|
||||
def _on_progress(self, task_id: str, stage: str) -> None:
|
||||
"""Ingester 阶段回调：推进任务状态并同步镜像（同步函数，供 progress_cb 使用）"""
|
||||
record = self._tasks.get(task_id)
|
||||
if record is None:
|
||||
return
|
||||
record["status"] = stage
|
||||
record["updated_at"] = _utc_now_iso()
|
||||
self._schedule_mirror(task_id)
|
||||
|
||||
def _schedule_mirror(self, task_id: str) -> None:
|
||||
"""将当前任务状态快照异步镜像到 Redis（同步上下文也可调用）"""
|
||||
if self._redis is None:
|
||||
return
|
||||
mirror = asyncio.create_task(self._mirror_to_redis(task_id, dict(self._tasks[task_id])))
|
||||
self._mirror_tasks.add(mirror)
|
||||
mirror.add_done_callback(self._mirror_tasks.discard)
|
||||
|
||||
async def _mirror_to_redis(self, task_id: str, snapshot: dict[str, Any]) -> None:
|
||||
"""写入 Redis 镜像：进行中与 done 用 ttl_done，failed 用 ttl_failed；写失败仅告警"""
|
||||
ttl = (
|
||||
self._settings.ingest_task_ttl_failed
|
||||
if snapshot["status"] == IngestTaskStatus.FAILED
|
||||
else self._settings.ingest_task_ttl_done
|
||||
)
|
||||
try:
|
||||
await self._redis.set_json(f"{REDIS_KEY_PREFIX}{task_id}", snapshot, ttl=ttl)
|
||||
except Exception:
|
||||
logger.warning("入库任务状态镜像 Redis 失败", task_id=task_id, exc_info=True)
|
||||
@@ -0,0 +1,306 @@
|
||||
"""文档入库模块
|
||||
|
||||
入库流程：文档输入 → 三级总结(Ollama) → 分类判定(L1总结) → 切分 chunk
|
||||
→ 构建 L2/L3 大纲节点 → 批量向量化（dense + sparse）→ 写入 Qdrant 四层集合
|
||||
|
||||
L2/L3 大纲节点的构建策略见 _build_l2_nodes / _build_l3_nodes。
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.core.chunker import Chunker
|
||||
from app.core.classifier import Classifier
|
||||
from app.core.embeddings import EmbeddingService, create_embedding_service
|
||||
from app.core.headings import Heading, parse_headings
|
||||
from app.core.sparse import SparseEncoder
|
||||
from app.core.summarizer import Summarizer
|
||||
from app.models.document import ChunkModel, DocumentInput, DocumentSummary, IngestionResult, SummaryLevel
|
||||
from app.models.knowledge import CategoryResult
|
||||
from app.services.qdrant import COLLECTION_L2, COLLECTION_L3, QdrantService, SparseVectorTuple
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# IngestionError 阶段标识
|
||||
STAGE_SUMMARIZE = "summarize"
|
||||
STAGE_CLASSIFY = "classify"
|
||||
STAGE_EMBED = "embed"
|
||||
STAGE_QDRANT = "qdrant"
|
||||
|
||||
|
||||
class IngestionError(Exception):
|
||||
"""入库失败异常
|
||||
|
||||
携带失败阶段（stage）与已产出的总结（summary，如有），
|
||||
Qdrant 写入失败时总结不丢，上层可按阶段重试。
|
||||
"""
|
||||
|
||||
def __init__(self, stage: str, message: str, summary: DocumentSummary | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.stage = stage
|
||||
self.summary = summary
|
||||
|
||||
|
||||
def _heading_paths(headings: list[Heading]) -> list[tuple[str, str]]:
|
||||
"""按文档顺序计算每个标题的 (标题文本, 祖先标题链含自身)，链用 " / " 连接"""
|
||||
paths: list[tuple[str, str]] = []
|
||||
stack: list[Heading] = []
|
||||
for heading in headings:
|
||||
# 遇到同级或更高级标题时弹栈，维护当前祖先链
|
||||
while stack and stack[-1].level >= heading.level:
|
||||
stack.pop()
|
||||
stack.append(heading)
|
||||
paths.append((heading.title, " / ".join(h.title for h in stack)))
|
||||
return paths
|
||||
|
||||
|
||||
def _split_l3_blocks(outline: str) -> list[tuple[str, str]]:
|
||||
"""将内容大纲按 "## " 行分块，返回 (块标题, 块文本) 列表
|
||||
|
||||
无 "## " 行时整块作为一个节点（块标题为空）；
|
||||
首个 "## " 之前的引导内容直接忽略。
|
||||
"""
|
||||
stripped = outline.strip()
|
||||
if not stripped:
|
||||
return []
|
||||
|
||||
blocks: list[tuple[str, list[str]]] = []
|
||||
for line in stripped.splitlines():
|
||||
if line.startswith("## "):
|
||||
blocks.append((line[3:].strip(), [line]))
|
||||
elif blocks:
|
||||
blocks[-1][1].append(line)
|
||||
if not blocks:
|
||||
return [("", stripped)]
|
||||
return [(title, "\n".join(lines).strip()) for title, lines in blocks]
|
||||
|
||||
|
||||
class Ingester:
|
||||
"""文档入库器：编排总结、分类、切分、向量化与 Qdrant 写入全链路"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
summarizer: Summarizer | None = None,
|
||||
classifier: Classifier | None = None,
|
||||
chunker: Chunker | None = None,
|
||||
embedding: EmbeddingService | None = None,
|
||||
sparse: SparseEncoder | None = None,
|
||||
qdrant: QdrantService | None = None,
|
||||
) -> None:
|
||||
self.summarizer = summarizer or Summarizer()
|
||||
self.classifier = classifier or Classifier()
|
||||
self.chunker = chunker or Chunker()
|
||||
self.embedding = embedding or create_embedding_service()
|
||||
self.sparse = sparse or SparseEncoder()
|
||||
self.qdrant = qdrant or QdrantService()
|
||||
|
||||
async def ingest(self, doc: DocumentInput, progress_cb: Callable[[str], None] | None = None) -> IngestionResult:
|
||||
"""执行文档入库
|
||||
|
||||
Args:
|
||||
doc: 文档输入（文本内容 + 元数据）
|
||||
progress_cb: 可选的阶段进度回调（同步函数），在各阶段边界以
|
||||
"summarizing" / "classifying" / "embedding" / "writing" 调用
|
||||
|
||||
Returns:
|
||||
IngestionResult: 入库结果
|
||||
|
||||
Raises:
|
||||
IngestionError: 任一阶段失败时抛出，携带 stage 与已产出总结
|
||||
"""
|
||||
|
||||
def _report(stage: str) -> None:
|
||||
if progress_cb is not None:
|
||||
progress_cb(stage)
|
||||
|
||||
logger.info("开始文档入库", title=doc.title, text_length=len(doc.text))
|
||||
doc_id = uuid.uuid4().hex
|
||||
|
||||
# 1. 三级总结
|
||||
_report("summarizing")
|
||||
try:
|
||||
summary = await self.summarizer.summarize(doc.text, title=doc.title)
|
||||
except Exception as exc:
|
||||
logger.error("入库失败：三级总结", stage=STAGE_SUMMARIZE, error=str(exc))
|
||||
raise IngestionError(STAGE_SUMMARIZE, f"三级总结失败: {exc}") from exc
|
||||
logger.info("三级总结完成", doc_id=doc_id, level=summary.level.value)
|
||||
|
||||
# 2. 分类判定（基于 L1 总结）
|
||||
_report("classifying")
|
||||
try:
|
||||
category = await self.classifier.classify(summary.l1_summary, title=doc.title)
|
||||
except Exception as exc:
|
||||
logger.error("入库失败：分类判定", stage=STAGE_CLASSIFY, error=str(exc))
|
||||
raise IngestionError(STAGE_CLASSIFY, f"分类判定失败: {exc}", summary=summary) from exc
|
||||
logger.info("分类判定完成", doc_id=doc_id, category=category.main_category, confidence=category.confidence)
|
||||
|
||||
# 3. 切分 chunk 并构建 L2/L3 大纲节点（(text, section_path) 列表）
|
||||
chunks = self.chunker.chunk(doc.text, doc_id)
|
||||
l2_nodes = self._build_l2_nodes(doc.text, summary)
|
||||
l3_nodes = self._build_l3_nodes(doc.text, summary)
|
||||
|
||||
# 4. 批量 embedding：L1 + L2 + L3 + chunks 一次调用，按序切片取向量
|
||||
texts = [
|
||||
summary.l1_summary,
|
||||
*(node_text for node_text, _ in l2_nodes),
|
||||
*(node_text for node_text, _ in l3_nodes),
|
||||
*(c.text for c in chunks),
|
||||
]
|
||||
try:
|
||||
_report("embedding")
|
||||
vectors = await self.embedding.embed(texts)
|
||||
except Exception as exc:
|
||||
logger.error("入库失败：向量化", stage=STAGE_EMBED, error=str(exc))
|
||||
raise IngestionError(STAGE_EMBED, f"向量化失败: {exc}", summary=summary) from exc
|
||||
l1_vector = vectors[0]
|
||||
l2_vectors = vectors[1 : 1 + len(l2_nodes)]
|
||||
l3_vectors = vectors[1 + len(l2_nodes) : 1 + len(l2_nodes) + len(l3_nodes)]
|
||||
chunk_vectors = vectors[1 + len(l2_nodes) + len(l3_nodes) :]
|
||||
|
||||
# 5. sparse 向量（仅 L1 与 chunks 需要）
|
||||
l1_sparse: SparseVectorTuple | None = None
|
||||
chunk_sparses: list[SparseVectorTuple | None] = [None] * len(chunks)
|
||||
if settings.sparse_enabled:
|
||||
l1_sparse = self.sparse.encode(summary.l1_summary)
|
||||
chunk_sparses = [self.sparse.encode(c.text) for c in chunks]
|
||||
|
||||
# 6. 写入 Qdrant 四层集合
|
||||
_report("writing")
|
||||
try:
|
||||
await self._write_qdrant(
|
||||
doc_id,
|
||||
doc,
|
||||
summary,
|
||||
category,
|
||||
chunks,
|
||||
l2_nodes,
|
||||
l3_nodes,
|
||||
l1_vector,
|
||||
l2_vectors,
|
||||
l3_vectors,
|
||||
chunk_vectors,
|
||||
l1_sparse,
|
||||
chunk_sparses,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("入库失败：Qdrant 写入", stage=STAGE_QDRANT, doc_id=doc_id, error=str(exc))
|
||||
raise IngestionError(STAGE_QDRANT, f"Qdrant 写入失败: {exc}", summary=summary) from exc
|
||||
|
||||
logger.info(
|
||||
"文档入库完成",
|
||||
doc_id=doc_id,
|
||||
chunks_count=len(chunks),
|
||||
l2_nodes=len(l2_nodes),
|
||||
l3_nodes=len(l3_nodes),
|
||||
category=category.main_category,
|
||||
)
|
||||
return IngestionResult(
|
||||
document_id=doc_id,
|
||||
summary=summary,
|
||||
category=category.main_category,
|
||||
tags=category.tags,
|
||||
category_confidence=category.confidence,
|
||||
collection="四层集合",
|
||||
chunks_count=len(chunks),
|
||||
)
|
||||
|
||||
def _build_l2_nodes(self, text: str, summary: DocumentSummary) -> list[tuple[str, str]]:
|
||||
"""构建 L2 大纲节点，返回 (text, section_path) 列表
|
||||
|
||||
- 有标题结构（L3 级且标题数 >= 2）：每个标题一个节点，
|
||||
text 与 section_path 均为该节点的祖先标题链
|
||||
- 否则若 l2_outline 非空（LLM 生成的大纲）：按非空行拆节点，section_path 为空
|
||||
- 2.5 级文档（l2_outline 为 None）：无 L2 节点
|
||||
"""
|
||||
headings = parse_headings(text)
|
||||
if summary.level == SummaryLevel.L3 and len(headings) >= 2:
|
||||
return [(path, path) for _, path in _heading_paths(headings)]
|
||||
if summary.l2_outline:
|
||||
return [(line.strip(), "") for line in summary.l2_outline.splitlines() if line.strip()]
|
||||
return []
|
||||
|
||||
def _build_l3_nodes(self, text: str, summary: DocumentSummary) -> list[tuple[str, str]]:
|
||||
"""构建 L3 内容大纲节点，返回 (text, section_path) 列表
|
||||
|
||||
按 "## " 分块（无 "## " 则整块一个节点）；
|
||||
section_path 尽力匹配文档标题链（块标题与文档标题文本精确匹配），匹配不到用 ""。
|
||||
"""
|
||||
path_by_title: dict[str, str] = {}
|
||||
for title, path in _heading_paths(parse_headings(text)):
|
||||
path_by_title.setdefault(title, path)
|
||||
return [
|
||||
(block_text, path_by_title.get(block_title, ""))
|
||||
for block_title, block_text in _split_l3_blocks(summary.l3_content_outline)
|
||||
]
|
||||
|
||||
async def _write_qdrant(
|
||||
self,
|
||||
doc_id: str,
|
||||
doc: DocumentInput,
|
||||
summary: DocumentSummary,
|
||||
category: CategoryResult,
|
||||
chunks: list[ChunkModel],
|
||||
l2_nodes: list[tuple[str, str]],
|
||||
l3_nodes: list[tuple[str, str]],
|
||||
l1_vector: list[float],
|
||||
l2_vectors: list[list[float]],
|
||||
l3_vectors: list[list[float]],
|
||||
chunk_vectors: list[list[float]],
|
||||
l1_sparse: SparseVectorTuple | None,
|
||||
chunk_sparses: list[SparseVectorTuple | None],
|
||||
) -> None:
|
||||
"""将 L1/L2/L3/chunks 四层数据写入 Qdrant（任一失败向上抛出）"""
|
||||
await self.qdrant.upsert_l1(
|
||||
doc_id=doc_id,
|
||||
title=doc.title,
|
||||
summary=summary.l1_summary,
|
||||
category=category.main_category,
|
||||
tags=category.tags,
|
||||
dense_vector=l1_vector,
|
||||
sparse_vector=l1_sparse,
|
||||
)
|
||||
|
||||
# L2/L3 大纲节点（为空时跳过对应集合的 upsert）
|
||||
for collection, nodes, vectors in (
|
||||
(COLLECTION_L2, l2_nodes, l2_vectors),
|
||||
(COLLECTION_L3, l3_nodes, l3_vectors),
|
||||
):
|
||||
if not nodes:
|
||||
continue
|
||||
await self.qdrant.upsert_nodes(
|
||||
collection,
|
||||
[
|
||||
{
|
||||
"doc_id": doc_id,
|
||||
"section_path": section_path,
|
||||
"text": node_text,
|
||||
"category": category.main_category,
|
||||
"tags": category.tags,
|
||||
"dense_vector": vector,
|
||||
}
|
||||
for (node_text, section_path), vector in zip(nodes, vectors, strict=True)
|
||||
],
|
||||
)
|
||||
|
||||
if chunks:
|
||||
# chunk dict 额外携带 doc_summary（= L1 总结），检索侧直接取用，不用回查 L1
|
||||
chunk_dicts: list[dict[str, Any]] = [
|
||||
{
|
||||
"doc_id": doc_id,
|
||||
"chunk_index": chunk.chunk_index,
|
||||
"text": chunk.text,
|
||||
"section_path": chunk.section_path,
|
||||
"title": doc.title,
|
||||
"category": category.main_category,
|
||||
"tags": category.tags,
|
||||
"dense_vector": vector,
|
||||
"sparse_vector": sparse,
|
||||
"doc_summary": summary.l1_summary,
|
||||
}
|
||||
for chunk, vector, sparse in zip(chunks, chunk_vectors, chunk_sparses, strict=True)
|
||||
]
|
||||
await self.qdrant.upsert_chunks(chunk_dicts)
|
||||
@@ -0,0 +1,214 @@
|
||||
"""query 解析与分类路由模块
|
||||
|
||||
分层 RAG 在线侧第一步：用 Ollama 小模型将用户 query 解析为结构化 JSON
|
||||
（命中类目+置信度、rewrite 后 query、关键词），再由纯函数做路由决策：
|
||||
- 高置信且命中类目数 <= 上限 → 按类目过滤检索
|
||||
- 低置信 / 解析失败 / 命中类目过多 → 全库兜底（不丢召回）
|
||||
|
||||
解析（LLM 调用）与决策（纯函数）分离，便于单元测试。
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from hashlib import sha256
|
||||
|
||||
import structlog
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
from app.config import settings
|
||||
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.redis import RedisCache, get_cache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 从 LLM 输出中提取第一个 {...} JSON 块（贪婪匹配到最后的 }，兼容嵌套对象）
|
||||
_JSON_BLOCK_RE = re.compile(r"\{.*\}", re.DOTALL)
|
||||
|
||||
|
||||
class CategoryHit(BaseModel):
|
||||
"""query 命中的类目及置信度"""
|
||||
|
||||
name: str = Field(description="类目名称，必须来自 taxonomy")
|
||||
confidence: float = Field(ge=0, le=1, description="命中置信度，范围 [0, 1]")
|
||||
|
||||
|
||||
class ParsedQuery(BaseModel):
|
||||
"""query 解析结果"""
|
||||
|
||||
raw_query: str = Field(description="原始 query")
|
||||
rewrite: str = Field(description="rewrite 后的 query")
|
||||
keywords: list[str] = Field(default_factory=list, description="提取的关键词")
|
||||
categories: list[CategoryHit] = Field(default_factory=list, description="命中类目列表")
|
||||
parse_failed: bool = Field(default=False, description="LLM 输出解析是否失败")
|
||||
|
||||
|
||||
class RouteDecision(BaseModel):
|
||||
"""路由决策结果"""
|
||||
|
||||
fallback: bool = Field(description="是否走全库兜底")
|
||||
filter_categories: list[str] | None = Field(default=None, description="过滤类目名列表，None 表示全库不过滤")
|
||||
reason: str = Field(description="决策原因：parse_failed | low_confidence | too_many_categories | routed")
|
||||
parsed: ParsedQuery = Field(description="对应的 query 解析结果")
|
||||
|
||||
|
||||
def _extract_json(raw: str) -> dict | None:
|
||||
"""从 LLM 输出中提取 JSON 对象
|
||||
|
||||
先尝试直接解析；失败则用正则提取第一个 {...} 块再解析。
|
||||
返回 None 表示无法提取出合法的 JSON 对象。
|
||||
"""
|
||||
text = raw.strip()
|
||||
try:
|
||||
data = json.loads(text)
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
match = _JSON_BLOCK_RE.search(text)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def decide_route(parsed: ParsedQuery, threshold: float, max_categories: int) -> RouteDecision:
|
||||
"""路由决策（纯函数）
|
||||
|
||||
- 解析失败 → 全库兜底（parse_failed）
|
||||
- 无 confidence >= threshold 的类目 → 全库兜底（low_confidence）
|
||||
- 命中类目数 > max_categories → 全库兜底（too_many_categories）
|
||||
- 否则按类目过滤检索，类目按 confidence 降序排列（routed）
|
||||
"""
|
||||
if parsed.parse_failed:
|
||||
return RouteDecision(fallback=True, filter_categories=None, reason="parse_failed", parsed=parsed)
|
||||
|
||||
hits = [c for c in parsed.categories if c.confidence >= threshold]
|
||||
if not hits:
|
||||
return RouteDecision(fallback=True, filter_categories=None, reason="low_confidence", parsed=parsed)
|
||||
|
||||
if len(hits) > max_categories:
|
||||
return RouteDecision(fallback=True, filter_categories=None, reason="too_many_categories", parsed=parsed)
|
||||
|
||||
hits.sort(key=lambda c: c.confidence, reverse=True)
|
||||
return RouteDecision(
|
||||
fallback=False,
|
||||
filter_categories=[c.name for c in hits],
|
||||
reason="routed",
|
||||
parsed=parsed,
|
||||
)
|
||||
|
||||
|
||||
class QueryParser:
|
||||
"""query 解析器：调用 Ollama 小模型将 query 解析为结构化 JSON"""
|
||||
|
||||
def __init__(
|
||||
self, ollama: OllamaClient, taxonomy: list[TaxonomyCategory], cache: RedisCache | None = None
|
||||
) -> None:
|
||||
self.ollama = ollama
|
||||
self.taxonomy = taxonomy
|
||||
# 解析结果缓存，缺省用全局单例；RedisCache 全操作容错，缓存不可用时退化为无缓存行为
|
||||
self.cache = cache if cache is not None else get_cache()
|
||||
# 可作为路由命中类目的名字集合（uncategorized 不可作为路由命中类目）
|
||||
self._routable_names = {c.name for c in taxonomy if c.name != UNCATEGORIZED}
|
||||
|
||||
async def parse(self, query: str) -> ParsedQuery:
|
||||
"""调用 LLM 将 query 解析为结构化结果
|
||||
|
||||
解析失败（输出非 JSON / 必填字段缺失或类型错误）时，
|
||||
返回 parse_failed=True 的兜底结果（rewrite 为原 query）。
|
||||
"""
|
||||
prompt = self._build_prompt(query)
|
||||
raw = await self.ollama.generate(prompt, json_mode=True)
|
||||
|
||||
data = _extract_json(raw)
|
||||
if data is None:
|
||||
logger.warning("query 解析失败：LLM 输出非合法 JSON", query=query, output=raw[:200])
|
||||
return ParsedQuery(raw_query=query, rewrite=query, parse_failed=True)
|
||||
|
||||
rewrite = data.get("rewrite")
|
||||
keywords = data.get("keywords")
|
||||
categories = data.get("categories")
|
||||
if not isinstance(rewrite, str) or not isinstance(keywords, list) or not isinstance(categories, list):
|
||||
logger.warning("query 解析失败：JSON 必填字段缺失或类型错误", query=query, output=raw[:200])
|
||||
return ParsedQuery(raw_query=query, rewrite=query, parse_failed=True)
|
||||
|
||||
return ParsedQuery(
|
||||
raw_query=query,
|
||||
rewrite=rewrite,
|
||||
keywords=[str(k) for k in keywords],
|
||||
categories=self._validate_categories(categories, query),
|
||||
)
|
||||
|
||||
async def parse_and_route(self, query: str) -> RouteDecision:
|
||||
"""解析 query 并做路由决策（threshold / max_categories 取自 settings）
|
||||
|
||||
parse() 的 LLM 解析结果按 query 哈希缓存；路由决策每次现算（纯函数，
|
||||
阈值取最新 settings），缓存数据损坏时回退为重新走 LLM 解析。
|
||||
"""
|
||||
cache_key = f"qparse:{sha256(query.encode()).hexdigest()[:16]}"
|
||||
parsed = await self._get_cached_parsed(cache_key, query)
|
||||
if parsed is None:
|
||||
parsed = await self.parse(query)
|
||||
await self.cache.set_json(cache_key, parsed.model_dump())
|
||||
return decide_route(
|
||||
parsed,
|
||||
threshold=settings.classify_confidence_threshold,
|
||||
max_categories=settings.classify_max_categories,
|
||||
)
|
||||
|
||||
async def _get_cached_parsed(self, cache_key: str, query: str) -> ParsedQuery | None:
|
||||
"""读取缓存的解析结果；未命中或缓存数据无法重建时返回 None"""
|
||||
cached = await self.cache.get_json(cache_key)
|
||||
if cached is None:
|
||||
return None
|
||||
try:
|
||||
parsed = ParsedQuery.model_validate(cached)
|
||||
except ValidationError:
|
||||
logger.warning("query 解析缓存数据损坏，按未命中处理", query=query, cache_key=cache_key)
|
||||
return None
|
||||
logger.info("query 解析缓存命中", query=query, cache_key=cache_key)
|
||||
return parsed
|
||||
|
||||
def _build_prompt(self, query: str) -> str:
|
||||
"""构造解析 prompt：列出 taxonomy 类目名+描述，要求模型只输出 JSON"""
|
||||
category_lines = [f"- {c.name}: {c.description}" for c in self.taxonomy if c.name != UNCATEGORIZED]
|
||||
category_block = "\n".join(category_lines)
|
||||
return (
|
||||
"你是搜索查询分析助手。请分析用户 query，完成三件事：\n"
|
||||
"1. 判断 query 意图命中以下哪些知识类目，并给出每个类目的置信度（0~1 之间的小数）；\n"
|
||||
"2. 将 query 改写为更适合检索的形式；\n"
|
||||
"3. 提取 query 的关键词。\n\n"
|
||||
f"可选类目：\n{category_block}\n\n"
|
||||
"要求：\n"
|
||||
"- categories 中的 name 只能从上面的类目名中选择，不要输出其他名称；\n"
|
||||
"- 若没有明显命中的类目，categories 返回空列表；\n"
|
||||
"- 只输出 JSON，不要输出任何其他内容。\n\n"
|
||||
'输出格式：{"categories": [{"name": "类目名", "confidence": 0.0}], '
|
||||
'"rewrite": "改写后的 query", "keywords": ["关键词"]}\n\n'
|
||||
f"用户 query：{query}"
|
||||
)
|
||||
|
||||
def _validate_categories(self, categories: list, query: str) -> list[CategoryHit]:
|
||||
"""校验并过滤 LLM 输出的类目条目
|
||||
|
||||
丢弃：类目名不在 taxonomy 可路由类目中的条目（含 uncategorized）、
|
||||
confidence 缺失或越界（不在 [0, 1]）的条目。
|
||||
"""
|
||||
hits: list[CategoryHit] = []
|
||||
for item in categories:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = item.get("name")
|
||||
confidence = item.get("confidence")
|
||||
if not isinstance(name, str) or name not in self._routable_names:
|
||||
logger.warning("丢弃不在 taxonomy 中的类目", query=query, name=name)
|
||||
continue
|
||||
if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
logger.warning("丢弃 confidence 越界的类目", query=query, name=name, confidence=confidence)
|
||||
continue
|
||||
hits.append(CategoryHit(name=name, confidence=float(confidence)))
|
||||
return hits
|
||||
@@ -0,0 +1,24 @@
|
||||
"""检索结果重排：RRF 融合与最终截断"""
|
||||
|
||||
from qdrant_client import models
|
||||
|
||||
|
||||
def rrf_fuse(result_lists: list[list[models.ScoredPoint]], k: int = 60) -> list[models.ScoredPoint]:
|
||||
"""标准 RRF（Reciprocal Rank Fusion）融合
|
||||
|
||||
融合分 = Σ 1/(k + rank)（rank 从 1 开始），按 point id 去重合并，
|
||||
返回按融合分降序的列表，score 字段写回融合分。空输入返回 []。
|
||||
"""
|
||||
scores: dict[str | int, float] = {}
|
||||
points: dict[str | int, models.ScoredPoint] = {}
|
||||
for results in result_lists:
|
||||
for rank, point in enumerate(results, start=1):
|
||||
scores[point.id] = scores.get(point.id, 0.0) + 1.0 / (k + rank)
|
||||
points.setdefault(point.id, point)
|
||||
ordered = sorted(points, key=lambda pid: scores[pid], reverse=True)
|
||||
return [points[pid].model_copy(update={"score": scores[pid]}) for pid in ordered]
|
||||
|
||||
|
||||
def finalize(points: list[models.ScoredPoint], final_k: int) -> list[models.ScoredPoint]:
|
||||
"""截断为最终返回的 top final_k"""
|
||||
return points[:final_k]
|
||||
@@ -0,0 +1,169 @@
|
||||
"""分层检索引擎
|
||||
|
||||
L1（文档总结）→ L2（章节大纲）→ L3（小节定位）→ chunks（原文）逐层收窄：
|
||||
- L1 无候选文档：直接全库 chunk 兜底，fallback=True
|
||||
- L2 无命中：全部候选文档回退到 L3 的 doc 级查询（b 路）
|
||||
- 2.5 级文档无 L2 节点，天然落入 L3 b 路（仅按 doc 过滤）
|
||||
- L3 两路（a：L2 命中文档按 section 过滤；b：其余文档仅按 doc 过滤）RRF 融合；
|
||||
L3 无命中时 chunk 层回退为 L1 候选文档级检索
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import structlog
|
||||
from qdrant_client import models
|
||||
|
||||
from app.config import settings
|
||||
from app.core.embeddings import EmbeddingService, create_embedding_service
|
||||
from app.core.query_parser import QueryParser
|
||||
from app.core.ranker import finalize, rrf_fuse
|
||||
from app.core.sparse import SparseEncoder
|
||||
from app.models.knowledge import load_taxonomy
|
||||
from app.models.search import SearchHit, SearchRequest, SearchResponse
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.qdrant import (
|
||||
COLLECTION_CHUNKS,
|
||||
COLLECTION_L1,
|
||||
COLLECTION_L2,
|
||||
COLLECTION_L3,
|
||||
SPARSE_COLLECTIONS,
|
||||
QdrantService,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
def _unique(values: Iterable[str | None]) -> list[str]:
|
||||
"""去重保序并丢弃空值"""
|
||||
return list(dict.fromkeys(v for v in values if v))
|
||||
|
||||
|
||||
class Retriever:
|
||||
"""分层检索引擎，依赖均可注入（默认自建，便于测试替换）"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
qdrant: QdrantService | None = None,
|
||||
query_parser: QueryParser | None = None,
|
||||
embedding: EmbeddingService | None = None,
|
||||
sparse_encoder: SparseEncoder | None = None,
|
||||
) -> None:
|
||||
self.qdrant = qdrant or QdrantService()
|
||||
self.query_parser = query_parser or QueryParser(
|
||||
ollama=OllamaClient(),
|
||||
taxonomy=load_taxonomy(settings.taxonomy_path),
|
||||
)
|
||||
self.embedding = embedding or create_embedding_service()
|
||||
self.sparse_encoder = sparse_encoder or SparseEncoder()
|
||||
|
||||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||||
"""分层检索主流程"""
|
||||
route = await self.query_parser.parse_and_route(request.query)
|
||||
query_text = route.parsed.rewrite or request.query
|
||||
dense = (await self.embedding.embed([query_text]))[0]
|
||||
sparse = self.sparse_encoder.encode(query_text) if settings.sparse_enabled else None
|
||||
# 路由兜底时不做类目过滤，避免丢召回
|
||||
categories = None if route.fallback else route.filter_categories
|
||||
|
||||
# L1：文档级检索，产出候选文档
|
||||
l1_filter = QdrantService.build_filter(categories=categories)
|
||||
l1_hits = await self._search_collection(COLLECTION_L1, dense, sparse, settings.l1_doc_top_n, l1_filter)
|
||||
logger.info("L1 检索完成", hits=len(l1_hits), categories=categories)
|
||||
|
||||
if not l1_hits:
|
||||
# L1 无候选文档 → 全库 chunk 兜底
|
||||
chunk_hits = await self._search_collection(COLLECTION_CHUNKS, dense, sparse, settings.retrieval_top_k, None)
|
||||
logger.info("L1 无命中，全库 chunk 兜底", hits=len(chunk_hits))
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
hits=self._to_hits(finalize(chunk_hits, self._final_k(request))),
|
||||
routed_categories=route.filter_categories or [],
|
||||
fallback=True,
|
||||
)
|
||||
|
||||
doc_ids = _unique((p.payload or {}).get("doc_id") for p in l1_hits)
|
||||
|
||||
# L2：候选文档内检索章节大纲
|
||||
l2_filter = QdrantService.build_filter(categories=categories, doc_ids=doc_ids)
|
||||
l2_hits = await self._search_collection(
|
||||
COLLECTION_L2, dense, sparse, settings.l2_section_top_n * len(doc_ids), l2_filter
|
||||
)
|
||||
logger.info("L2 检索完成", hits=len(l2_hits))
|
||||
l2_doc_ids = _unique((p.payload or {}).get("doc_id") for p in l2_hits)
|
||||
l2_section_paths = _unique((p.payload or {}).get("section_path") for p in l2_hits)
|
||||
# 无 L2 命中的文档（2.5 级文档或该 doc 的 L2 未命中）走 L3 b 路
|
||||
remaining_doc_ids = [d for d in doc_ids if d not in l2_doc_ids]
|
||||
|
||||
# L3：两路查询后 RRF 融合
|
||||
l3_lists: list[list[models.ScoredPoint]] = []
|
||||
if l2_doc_ids:
|
||||
l3_filter_a = QdrantService.build_filter(
|
||||
categories=categories, doc_ids=l2_doc_ids, section_paths=l2_section_paths
|
||||
)
|
||||
l3_lists.append(await self._search_collection(COLLECTION_L3, dense, sparse, settings.l3_top_n, l3_filter_a))
|
||||
if remaining_doc_ids:
|
||||
l3_filter_b = QdrantService.build_filter(categories=categories, doc_ids=remaining_doc_ids)
|
||||
l3_lists.append(await self._search_collection(COLLECTION_L3, dense, sparse, settings.l3_top_n, l3_filter_b))
|
||||
l3_hits = rrf_fuse(l3_lists)
|
||||
logger.info("L3 检索完成", hits=len(l3_hits))
|
||||
|
||||
# chunk 层：L3 有命中按 section 收窄；无命中回退为 L1 候选文档级检索
|
||||
if l3_hits:
|
||||
l3_doc_ids = _unique((p.payload or {}).get("doc_id") for p in l3_hits)
|
||||
# L3 命中 section_path 全为空（如 2.5 级文档）时仅按 doc 过滤
|
||||
l3_section_paths = _unique((p.payload or {}).get("section_path") for p in l3_hits)
|
||||
chunk_filter = QdrantService.build_filter(
|
||||
categories=categories, doc_ids=l3_doc_ids, section_paths=l3_section_paths
|
||||
)
|
||||
else:
|
||||
logger.info("L3 无命中，回退到 L1 候选文档级 chunk 检索")
|
||||
chunk_filter = QdrantService.build_filter(categories=categories, doc_ids=doc_ids)
|
||||
|
||||
chunk_hits = await self._search_collection(
|
||||
COLLECTION_CHUNKS, dense, sparse, settings.retrieval_top_k, chunk_filter
|
||||
)
|
||||
logger.info("chunk 检索完成", hits=len(chunk_hits))
|
||||
|
||||
final_points = finalize(rrf_fuse([chunk_hits]), self._final_k(request))
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
hits=self._to_hits(final_points),
|
||||
routed_categories=route.filter_categories or [],
|
||||
fallback=route.fallback,
|
||||
)
|
||||
|
||||
async def _search_collection(
|
||||
self,
|
||||
collection: str,
|
||||
dense: list[float],
|
||||
sparse: tuple[list[int], list[float]] | None,
|
||||
limit: int,
|
||||
query_filter: models.Filter | None,
|
||||
) -> list[models.ScoredPoint]:
|
||||
"""按集合是否支持 sparse 选择 hybrid 或 dense 检索"""
|
||||
if sparse is not None and collection in SPARSE_COLLECTIONS:
|
||||
return await self.qdrant.search_hybrid(collection, dense, sparse, limit, query_filter=query_filter)
|
||||
return await self.qdrant.search_dense(collection, dense, limit, query_filter=query_filter)
|
||||
|
||||
@staticmethod
|
||||
def _final_k(request: SearchRequest) -> int:
|
||||
"""最终返回数：请求指定优先，否则用配置默认值"""
|
||||
return request.top_k or settings.retrieval_final_k
|
||||
|
||||
@staticmethod
|
||||
def _to_hits(points: list[models.ScoredPoint]) -> list[SearchHit]:
|
||||
"""chunk 点组装为 SearchHit，字段取自 chunk payload（doc_summary 仅上下文标注）"""
|
||||
hits: list[SearchHit] = []
|
||||
for p in points:
|
||||
payload = p.payload or {}
|
||||
hits.append(
|
||||
SearchHit(
|
||||
text=payload.get("text", ""),
|
||||
doc_id=payload.get("doc_id", ""),
|
||||
title=payload.get("title", ""),
|
||||
section_path=payload.get("section_path") or "",
|
||||
score=p.score,
|
||||
doc_summary=payload.get("doc_summary") or "",
|
||||
)
|
||||
)
|
||||
return hits
|
||||
@@ -0,0 +1,68 @@
|
||||
"""BM25 轻量近似稀疏向量编码器
|
||||
|
||||
零第三方依赖的稀疏编码实现：无全局 IDF 的 BM25 近似（仅用词频 tf 加权），
|
||||
输出 Qdrant SparseVector 所需的 indices/values 格式。
|
||||
后续可替换为 SPLADE / BM42 等更强的稀疏编码器。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
|
||||
# 稀疏向量维度（哈希空间大小）
|
||||
SPARSE_DIM = 2**18
|
||||
|
||||
# 连续 CJK 字符段（一-鿿）
|
||||
_CJK_RE = re.compile(r"[一-鿿]+")
|
||||
# CJK 段或英文/数字连续段
|
||||
_TOKEN_RE = re.compile(r"[一-鿿]+|[a-zA-Z0-9]+")
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[str]:
|
||||
"""分词（纯正则实现）
|
||||
|
||||
- 连续 CJK 字符段：长度 1 时保留 unigram，长度 >= 2 时生成字符 bigram
|
||||
- 英文/数字连续段：小写化后作为整词
|
||||
"""
|
||||
tokens: list[str] = []
|
||||
for match in _TOKEN_RE.finditer(text):
|
||||
seg = match.group()
|
||||
if _CJK_RE.fullmatch(seg):
|
||||
if len(seg) == 1:
|
||||
tokens.append(seg)
|
||||
else:
|
||||
tokens.extend(seg[i : i + 2] for i in range(len(seg) - 1))
|
||||
else:
|
||||
tokens.append(seg.lower())
|
||||
return tokens
|
||||
|
||||
|
||||
def _hash(token: str) -> int:
|
||||
"""将词哈希到 [0, SPARSE_DIM) 的索引空间"""
|
||||
digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
|
||||
return int.from_bytes(digest, "big") % SPARSE_DIM
|
||||
|
||||
|
||||
class SparseEncoder:
|
||||
"""稀疏向量编码器，输出 Qdrant SparseVector 的 indices/values"""
|
||||
|
||||
def encode(self, text: str) -> tuple[list[int], list[float]]:
|
||||
"""编码单条文本
|
||||
|
||||
权重 = 1 + log(tf)，词经哈希到 [0, SPARSE_DIM)，哈希冲突时权重累加。
|
||||
|
||||
Returns:
|
||||
(indices, values)：indices 升序且无重复，与 values 等长
|
||||
"""
|
||||
# 按哈希索引累计词频，天然处理哈希冲突
|
||||
tf: dict[int, float] = {}
|
||||
for token in _tokenize(text):
|
||||
idx = _hash(token)
|
||||
tf[idx] = tf.get(idx, 0.0) + 1.0
|
||||
indices = sorted(tf)
|
||||
values = [1.0 + math.log(tf[idx]) for idx in indices]
|
||||
return indices, values
|
||||
|
||||
def encode_batch(self, texts: list[str]) -> list[tuple[list[int], list[float]]]:
|
||||
"""批量编码，与逐条 encode 结果一致"""
|
||||
return [self.encode(text) for text in texts]
|
||||
@@ -0,0 +1,138 @@
|
||||
"""文档三级总结模块
|
||||
|
||||
通过 Ollama 本地小模型对文档进行分级总结：
|
||||
- L1: 总结（一句话高度概括）
|
||||
- L2: 大纲（主要章节和关键主题）
|
||||
- L3: 内容大纲（每个章节的详细内容摘要）
|
||||
|
||||
当文档内容不足以支撑三级总结时，自动降级为 2.5 级（L1 + L2.5 内容大纲）。
|
||||
"""
|
||||
|
||||
import structlog
|
||||
|
||||
from app.core.headings import parse_headings, render_outline
|
||||
from app.models.document import DocumentSummary, SummaryLevel
|
||||
from app.services.ollama import OllamaClient
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 文本长度阈值（字符数），低于此值触发 2.5 级回退
|
||||
MIN_TEXT_LENGTH_FOR_L3 = 500
|
||||
|
||||
|
||||
class Summarizer:
|
||||
"""文档三级总结器"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
|
||||
async def summarize(self, text: str, *, title: str = "") -> DocumentSummary:
|
||||
"""对文档文本进行三级总结
|
||||
|
||||
Args:
|
||||
text: 文档纯文本内容
|
||||
title: 文档标题（可选，辅助总结）
|
||||
|
||||
Returns:
|
||||
DocumentSummary: 包含各级总结的结果
|
||||
"""
|
||||
text_length = len(text.strip())
|
||||
|
||||
# 生成 L1 总结
|
||||
l1_summary = await self._generate_l1(text, title)
|
||||
logger.info("L1 总结完成", title=title, l1=l1_summary[:100])
|
||||
|
||||
# 判断是否需要 2.5 级回退
|
||||
use_fallback = text_length < MIN_TEXT_LENGTH_FOR_L3
|
||||
|
||||
if use_fallback:
|
||||
logger.info("文档内容不足，使用 2.5 级回退", text_length=text_length)
|
||||
# L2.5: 直接生成内容大纲（跳过大纲层）
|
||||
l2_half = await self._generate_l2_half(text, title, l1_summary)
|
||||
return DocumentSummary(
|
||||
l1_summary=l1_summary,
|
||||
l2_outline=None,
|
||||
l3_content_outline=l2_half,
|
||||
level=SummaryLevel.L2_HALF,
|
||||
)
|
||||
|
||||
# 生成 L2 大纲：优先使用文档原生标题树（结构导航，不调 LLM），
|
||||
# 无结构文本（标题数 < 2）回退 LLM 生成
|
||||
headings = parse_headings(text)
|
||||
if len(headings) >= 2:
|
||||
l2_outline = render_outline(headings)
|
||||
logger.info("L2 大纲完成", title=title, outline_source="headings", heading_count=len(headings))
|
||||
else:
|
||||
l2_outline = await self._generate_l2(text, title, l1_summary)
|
||||
logger.info("L2 大纲完成", title=title, outline_source="llm")
|
||||
|
||||
# 生成 L3 内容大纲
|
||||
l3_content_outline = await self._generate_l3(text, title, l1_summary, l2_outline)
|
||||
logger.info("L3 内容大纲完成", title=title)
|
||||
|
||||
return DocumentSummary(
|
||||
l1_summary=l1_summary,
|
||||
l2_outline=l2_outline,
|
||||
l3_content_outline=l3_content_outline,
|
||||
level=SummaryLevel.L3,
|
||||
)
|
||||
|
||||
async def _generate_l1(self, text: str, title: str) -> str:
|
||||
"""生成 L1 总结：一句话高度概括"""
|
||||
prompt = (
|
||||
"请用一句话对以下文档内容进行高度概括，要求简洁精炼，"
|
||||
"突出文档的核心主题和关键信息。\n\n"
|
||||
)
|
||||
if title:
|
||||
prompt += f"文档标题：{title}\n\n"
|
||||
prompt += f"文档内容：\n{text}"
|
||||
|
||||
return await self.ollama.generate(prompt)
|
||||
|
||||
async def _generate_l2(self, text: str, title: str, l1_summary: str) -> str:
|
||||
"""生成 L2 大纲：主要章节和关键主题"""
|
||||
prompt = (
|
||||
"请提取以下文档的主要章节结构和关键主题，"
|
||||
"以大纲形式呈现。每个主题用一行表示，"
|
||||
"格式为：序号. 主题名称\n\n"
|
||||
f"文档总结：{l1_summary}\n\n"
|
||||
)
|
||||
if title:
|
||||
prompt += f"文档标题：{title}\n\n"
|
||||
prompt += f"文档内容：\n{text}"
|
||||
|
||||
return await self.ollama.generate(prompt)
|
||||
|
||||
async def _generate_l3(
|
||||
self, text: str, title: str, l1_summary: str, l2_outline: str
|
||||
) -> str:
|
||||
"""生成 L3 内容大纲：每个章节的详细内容摘要"""
|
||||
prompt = (
|
||||
"请对以下文档的每个章节/主题进行详细的内容摘要，"
|
||||
"格式为：\n"
|
||||
"## 章节名称\n"
|
||||
"详细摘要内容...\n\n"
|
||||
f"文档总结：{l1_summary}\n\n"
|
||||
f"文档大纲：\n{l2_outline}\n\n"
|
||||
)
|
||||
if title:
|
||||
prompt += f"文档标题：{title}\n\n"
|
||||
prompt += f"文档内容：\n{text}"
|
||||
|
||||
return await self.ollama.generate(prompt)
|
||||
|
||||
async def _generate_l2_half(self, text: str, title: str, l1_summary: str) -> str:
|
||||
"""生成 L2.5 内容大纲（2.5 级回退）
|
||||
|
||||
跳过大纲层，直接对短文本生成详细摘要。
|
||||
"""
|
||||
prompt = (
|
||||
"请对以下文档内容进行详细摘要，"
|
||||
"涵盖所有关键信息点。以要点形式呈现。\n\n"
|
||||
f"文档总结：{l1_summary}\n\n"
|
||||
)
|
||||
if title:
|
||||
prompt += f"文档标题：{title}\n\n"
|
||||
prompt += f"文档内容：\n{text}"
|
||||
|
||||
return await self.ollama.generate(prompt)
|
||||
Reference in New Issue
Block a user