feat: 完成全量功能开发,包括前端管理后台与后端服务优化
此提交实现了完整的知识库管理系统: 1. 新增Vue3 + Antd Vue前端管理后台,包含登录、文档管理、检索、类目设置等完整页面 2. 重构后端LLM调用抽象层,支持Ollama与OpenAI兼容服务动态切换 3. 调整默认嵌入模型配置为本地bge-m3模式 4. 优化入库任务去重逻辑与缓存清理机制 5. 完善Docker镜像构建与docker-compose部署配置 6. 修复多项测试用例与兼容性问题 7. 新增运行时配置API,支持动态调整系统参数
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
"""文档分类器
|
||||
|
||||
入库链路第二步:基于 L1 总结,用 Ollama 小模型将文档判定为 taxonomy 中的
|
||||
入库链路第二步:基于 L1 总结,用 LLM(Ollama 或 OpenAI 兼容服务,由
|
||||
runtime_settings.models.classify 决定)将文档判定为 taxonomy 中的
|
||||
主类目 + 附加标签:
|
||||
- LLM 输出解析失败 / 类目名不在 taxonomy → 归 uncategorized(confidence=0.0)
|
||||
- 置信度低于阈值 → 主类目归 uncategorized,候选类目名保留进 tags(软召回用)
|
||||
@@ -13,7 +14,7 @@ import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.models.knowledge import UNCATEGORIZED, CategoryResult, TaxonomyCategory, load_taxonomy
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
@@ -47,8 +48,10 @@ def _extract_json(raw: str) -> dict | None:
|
||||
class Classifier:
|
||||
"""文档分类器:将 L1 总结判定为 taxonomy 主类目 + 附加标签"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
def __init__(self, ollama: LLMClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
|
||||
# 默认按 runtime_settings.models.classify 选择 LLM 实现;
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("classify")
|
||||
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
|
||||
# 合法类目名集合(含 uncategorized)
|
||||
self._valid_names = {c.name for c in self.taxonomy}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""文本去重策略工厂
|
||||
|
||||
支持三种策略:
|
||||
- none:关闭去重,每次都跑完整流水线
|
||||
- sha256:精确匹配(hash 完全相同才算重复)
|
||||
- simhash:近似匹配(海明距离 <= 阈值视为重复)
|
||||
|
||||
每种策略实现 DedupStrategy 协议:lookup(text) -> dict|None、record(text, result) -> None。
|
||||
工厂 get_dedup_strategy() 根据 runtime_settings.dedup 选择实现。
|
||||
所有策略共享同一 Redis key 前缀,但 key 后缀按策略区分避免冲突。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
import structlog
|
||||
|
||||
from app.core.runtime_settings import get_runtime_settings
|
||||
from app.services.redis import RedisCache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# Redis 去重 key 前缀,按策略分桶
|
||||
_DEDUP_KEY_PREFIX = "dedup:"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DedupStrategy(Protocol):
|
||||
"""去重策略协议"""
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
"""查询文本是否已入库;命中返回旧 IngestionResult(dict),未命中返回 None"""
|
||||
...
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
"""记录文本已入库,供后续命中复用"""
|
||||
...
|
||||
|
||||
|
||||
class NoopDedupStrategy:
|
||||
"""关闭去重:永远不命中,也不记录"""
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class Sha256DedupStrategy:
|
||||
"""SHA256 精确去重:完全相同文本才算重复"""
|
||||
|
||||
def __init__(self, redis: RedisCache, ttl_seconds: int) -> None:
|
||||
self._redis = redis
|
||||
self._ttl = ttl_seconds
|
||||
|
||||
def _key(self, text: str) -> str:
|
||||
h = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
return f"{_DEDUP_KEY_PREFIX}sha256:{h}"
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
return await self._redis.get_json(self._key(text))
|
||||
except Exception:
|
||||
logger.warning("sha256 去重查询失败,降级未命中", exc_info=True)
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._redis.set_json(self._key(text), result_dict, ttl=self._ttl)
|
||||
except Exception:
|
||||
logger.warning("sha256 去重记录写入失败", exc_info=True)
|
||||
|
||||
|
||||
# SimHash 实现(64 位)
|
||||
_MASK_64 = (1 << 64) - 1
|
||||
|
||||
|
||||
def _simhash(text: str, token_size: int = 4) -> int:
|
||||
"""计算文本 64 位 simhash
|
||||
|
||||
简化实现:按 token_size 字符滑窗分词,每段 md5 → 128 位 → 取低 64 位作为 hash,
|
||||
逐位加权(hash 该位为 1 则 +1,为 0 则 -1),最终符号位定 1/0。
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
tokens = [text[i : i + token_size] for i in range(0, len(text), token_size)]
|
||||
weights = [1] * 64
|
||||
vec = [0] * 64
|
||||
for token in tokens:
|
||||
h = int(hashlib.md5(token.encode("utf-8")).hexdigest(), 16) & _MASK_64
|
||||
for i in range(64):
|
||||
bit = (h >> i) & 1
|
||||
vec[i] += weights[i] if bit else -weights[i]
|
||||
fingerprint = 0
|
||||
for i in range(64):
|
||||
if vec[i] > 0:
|
||||
fingerprint |= (1 << i)
|
||||
return fingerprint
|
||||
|
||||
|
||||
def _hamming_distance(a: int, b: int) -> int:
|
||||
return bin((a ^ b) & _MASK_64).count("1")
|
||||
|
||||
|
||||
class SimhashDedupStrategy:
|
||||
"""SimHash 近似去重:海明距离 <= 阈值视为重复
|
||||
|
||||
Redis 中存所有已入库文本的 simhash(value 为 IngestionResult + simhash)。
|
||||
lookup 时遍历所有候选 simhash 算海明距离,找最近的一个 <= 阈值则命中。
|
||||
注:本实现为简化版,全表扫描,适合中小规模知识库;超大规模需换 LSH 索引。
|
||||
"""
|
||||
|
||||
INDEX_KEY = f"{_DEDUP_KEY_PREFIX}simhash:index" # list 形式存所有 simhash+key
|
||||
|
||||
def __init__(self, redis: RedisCache, ttl_seconds: int, threshold: int = 3) -> None:
|
||||
self._redis = redis
|
||||
self._ttl = ttl_seconds
|
||||
self._threshold = max(0, min(64, threshold))
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
fingerprint = _simhash(text)
|
||||
# 简化:扫描所有 simhash 记录找最近的
|
||||
# 用 set:dedup:simhash:fps 存所有 fingerprint,每个 fp 对应一个 record key
|
||||
# 这里用 list key 简化(适合小规模)
|
||||
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
|
||||
entries = index.get("entries", [])
|
||||
best_dist = self._threshold + 1
|
||||
best_key: str | None = None
|
||||
for entry in entries:
|
||||
fp = entry.get("fingerprint")
|
||||
key = entry.get("key")
|
||||
if fp is None or key is None:
|
||||
continue
|
||||
dist = _hamming_distance(fingerprint, int(fp))
|
||||
if dist <= self._threshold and dist < best_dist:
|
||||
best_dist = dist
|
||||
best_key = key
|
||||
if best_key is None:
|
||||
return None
|
||||
return await self._redis.get_json(best_key)
|
||||
except Exception:
|
||||
logger.warning("simhash 去重查询失败,降级未命中", exc_info=True)
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
try:
|
||||
fingerprint = _simhash(text)
|
||||
key = f"{_DEDUP_KEY_PREFIX}simhash:{fingerprint:016x}"
|
||||
# 1. 写记录
|
||||
await self._redis.set_json(key, result_dict, ttl=self._ttl)
|
||||
# 2. 更新索引
|
||||
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
|
||||
entries = index.get("entries", [])
|
||||
entries.append({"fingerprint": fingerprint, "key": key})
|
||||
await self._redis.set_json(self.INDEX_KEY, {"entries": entries}, ttl=self._ttl)
|
||||
except Exception:
|
||||
logger.warning("simhash 去重记录写入失败", exc_info=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_strategy_cache: DedupStrategy | None = None
|
||||
_strategy_signature: tuple[str, int, int] | None = None # (strategy, ttl, threshold)
|
||||
|
||||
|
||||
def get_dedup_strategy(redis: RedisCache | None) -> DedupStrategy:
|
||||
"""按 runtime_settings.dedup 构造去重策略
|
||||
|
||||
Redis 不可用时降级为 NoopDedupStrategy(关闭去重),不影响主流程。
|
||||
策略配置变更时自动重建单例。
|
||||
"""
|
||||
global _strategy_cache, _strategy_signature
|
||||
|
||||
rt = get_runtime_settings()
|
||||
sig = (rt.dedup.strategy, rt.dedup.ttl_seconds, rt.dedup.simhash_threshold)
|
||||
|
||||
if _strategy_cache is not None and _strategy_signature == sig:
|
||||
return _strategy_cache
|
||||
|
||||
if redis is None or rt.dedup.strategy == "none":
|
||||
strategy: DedupStrategy = NoopDedupStrategy()
|
||||
elif rt.dedup.strategy == "sha256":
|
||||
strategy = Sha256DedupStrategy(redis=redis, ttl_seconds=rt.dedup.ttl_seconds)
|
||||
elif rt.dedup.strategy == "simhash":
|
||||
strategy = SimhashDedupStrategy(
|
||||
redis=redis, ttl_seconds=rt.dedup.ttl_seconds, threshold=rt.dedup.simhash_threshold
|
||||
)
|
||||
else:
|
||||
logger.warning("未知去重策略,降级为 noop", requested=rt.dedup.strategy)
|
||||
strategy = NoopDedupStrategy()
|
||||
|
||||
_strategy_cache = strategy
|
||||
_strategy_signature = sig
|
||||
return strategy
|
||||
|
||||
|
||||
def invalidate_dedup_strategy_cache() -> None:
|
||||
"""清除策略缓存(runtime_settings 更新后调用)"""
|
||||
global _strategy_cache, _strategy_signature
|
||||
_strategy_cache = None
|
||||
_strategy_signature = None
|
||||
|
||||
|
||||
def compute_text_hash(text: str) -> str:
|
||||
"""计算文本哈希(兼容旧接口,sha256 策略下与 dedup key 一致)"""
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
# 兼容旧测试与外部引用:保留 DEDUP_KEY_PREFIX 导出
|
||||
DEDUP_KEY_PREFIX = _DEDUP_KEY_PREFIX
|
||||
+346
-87
@@ -1,14 +1,19 @@
|
||||
"""多格式文件文本提取:按扩展名分发到对应解析器
|
||||
"""多格式文件文本提取:按扩展名分发到对应解析器(插件化)
|
||||
|
||||
支持的扩展名:
|
||||
- .txt / .md:UTF-8 解码(errors="replace" 兜底)
|
||||
- .html / .htm:标准库 html.parser 剥离标签提取可见文本
|
||||
- .pdf:pypdf 逐页 extract_text 拼接;文本层为空(扫描件/图片型)时
|
||||
自动降级为 OCR(pypdfium2 渲染 + rapidocr-onnxruntime 识别)
|
||||
- .docx:python-docx 段落文本拼接(不含表格/页眉页脚)
|
||||
- .pdf:PDF 文本层提取插件(默认 pypdf);文本层为空(扫描件)时降级到 OCR 插件
|
||||
- .docx:DOCX 解析插件(默认 python-docx)
|
||||
|
||||
插件化设计:
|
||||
- PdfTextExtractor / DocxParser / OcrEngine 三个 Protocol
|
||||
- 每种插件类型有注册表 + 默认实现 + 工厂
|
||||
- 工厂根据 runtime_settings.parsers 选择具体插件
|
||||
- 未注册或依赖缺失时降级到默认插件并告警
|
||||
|
||||
未识别扩展名抛 ValueError("不支持的文件类型: {ext}");
|
||||
解析异常统一包装为 ValueError("文件解析失败: {detail}"),原异常链式保留。
|
||||
解析异常统一包装为 ValueError("文件解析失败: {detail}")。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,17 +22,19 @@ import io
|
||||
from collections.abc import Callable
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.core.runtime_settings import get_runtime_settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 模块级懒加载 OCR 引擎单例(首次调用时初始化,避免无扫描件场景白白下载模型)
|
||||
_ocr_engine: Any | None = None
|
||||
_ocr_unavailable: bool = False # 标记 OCR 依赖不可用,后续直接跳过避免重复尝试
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# HTML / Text 解析(无插件化需求,保留原实现)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _VisibleTextExtractor(HTMLParser):
|
||||
@@ -51,13 +58,11 @@ class _VisibleTextExtractor(HTMLParser):
|
||||
self._parts.append(data)
|
||||
|
||||
def get_text(self) -> str:
|
||||
# 块级标签间用空格连接,再折叠多余空白
|
||||
text = " ".join(self._parts)
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def _decode_html(content: bytes) -> str:
|
||||
"""HTML 内容解码:优先 utf-8(带 BOM),失败回退 latin-1"""
|
||||
try:
|
||||
return content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
@@ -65,108 +70,362 @@ def _decode_html(content: bytes) -> str:
|
||||
|
||||
|
||||
def _parse_text(content: bytes) -> str:
|
||||
"""UTF-8 解码(errors=replace 兜底),保留原字符"""
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _parse_html(content: bytes) -> str:
|
||||
"""HTML 剥离标签,保留可见文本"""
|
||||
parser = _VisibleTextExtractor()
|
||||
parser.feed(_decode_html(content))
|
||||
parser.close()
|
||||
return parser.get_text()
|
||||
|
||||
|
||||
def _parse_pdf(content: bytes) -> str:
|
||||
"""PDF 解析:优先 pypdf extract_text;文本层为空(扫描件)时降级 OCR
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 插件协议
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
OCR 流程:pypdfium2 渲染每页为 PIL Image → rapidocr-onnxruntime 识别 →
|
||||
拼接每页识别出的文本。受 settings.pdf_ocr_* 控制:开关、最大页数、DPI。
|
||||
OCR 依赖未安装或运行异常时降级返回空字符串(由上游 upload 端点拒绝入库)。
|
||||
|
||||
@runtime_checkable
|
||||
class PdfTextExtractor(Protocol):
|
||||
"""PDF 文本层提取插件协议"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
"""从 PDF 二进制内容提取文本层;无文本层返回空字符串"""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DocxParser(Protocol):
|
||||
"""DOCX 解析插件协议"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
"""从 DOCX 二进制内容提取段落文本"""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class OcrEngine(Protocol):
|
||||
"""OCR 引擎插件协议"""
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
"""对 PDF 跑 OCR,返回识别文本;失败返回空字符串"""
|
||||
...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 PDF 插件:pypdf
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PypdfTextExtractor:
|
||||
"""pypdf 文本层提取(默认)"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(io.BytesIO(content))
|
||||
parts: list[str] = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 备选 PDF 插件:pdfplumber(lazy import,依赖缺失时不可用)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PdfplumberTextExtractor:
|
||||
"""pdfplumber 文本层提取(备选,对复杂排版更友好)"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
import pdfplumber # type: ignore[import-untyped]
|
||||
|
||||
parts: list[str] = []
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
for page in pdf.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 DOCX 插件:python-docx
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PythonDocxParser:
|
||||
"""python-docx 段落提取(默认)"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
from docx import Document # type: ignore[import-untyped]
|
||||
|
||||
document = Document(io.BytesIO(content))
|
||||
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 OCR 插件:rapidocr-onnxruntime + pypdfium2
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class RapidocrOcrEngine:
|
||||
"""rapidocr-onnxruntime OCR 引擎(默认)
|
||||
|
||||
流程:pypdfium2 渲染每页为 PIL Image → rapidocr 识别 → 拼接。
|
||||
类级懒加载单例(_engine/_unavailable 为类属性,跨实例共享),
|
||||
依赖缺失时降级返回空文本。
|
||||
"""
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(io.BytesIO(content))
|
||||
parts: list[str] = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
text_layer = "\n".join(parts).strip()
|
||||
_engine: Any | None = None
|
||||
_unavailable: bool = False
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
if self._unavailable:
|
||||
return ""
|
||||
|
||||
# 1. 懒加载 OCR 引擎(类级单例,跨实例共享)
|
||||
if self._engine is None:
|
||||
try:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
RapidocrOcrEngine._engine = RapidOCR()
|
||||
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
|
||||
except Exception:
|
||||
RapidocrOcrEngine._unavailable = True
|
||||
logger.warning(
|
||||
"OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
# 2. 渲染并识别
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
result, _ = self._engine(pil_image)
|
||||
if result:
|
||||
lines = [
|
||||
item[1]
|
||||
for item in result
|
||||
if item and len(item) >= 2 and item[1]
|
||||
]
|
||||
if lines:
|
||||
page_texts.append("\n".join(lines))
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 备选 OCR 插件:tesseract(lazy import,依赖缺失时不可用)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TesseractOcrEngine:
|
||||
"""pytesseract + pdfium OCR(备选)
|
||||
|
||||
需要系统安装 tesseract 二进制与语言包。
|
||||
类级 _unavailable 单例,跨实例共享。
|
||||
"""
|
||||
|
||||
_unavailable: bool = False
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
if self._unavailable:
|
||||
return ""
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
import pytesseract # type: ignore[import-untyped]
|
||||
from PIL import Image # type: ignore[import-untyped]
|
||||
except Exception:
|
||||
TesseractOcrEngine._unavailable = True
|
||||
logger.warning("tesseract 依赖不可用,降级返回空文本", exc_info=True)
|
||||
return ""
|
||||
|
||||
try:
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
text = pytesseract.image_to_string(pil_image, lang="chi_sim+eng")
|
||||
if text:
|
||||
page_texts.append(text)
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"tesseract OCR 失败,降级返回空文本", error=str(exc), exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
class NoopOcrEngine:
|
||||
"""关闭 OCR 的占位插件(strategy=none 时使用)"""
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 插件注册表
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_PDF_PLUGINS: dict[str, type[PdfTextExtractor]] = {
|
||||
"pypdf": PypdfTextExtractor,
|
||||
"pdfplumber": PdfplumberTextExtractor,
|
||||
}
|
||||
|
||||
_DOCX_PLUGINS: dict[str, type[DocxParser]] = {
|
||||
"python_docx": PythonDocxParser,
|
||||
}
|
||||
|
||||
_OCR_PLUGINS: dict[str, type[OcrEngine]] = {
|
||||
"rapidocr": RapidocrOcrEngine,
|
||||
"tesseract": TesseractOcrEngine,
|
||||
"none": NoopOcrEngine,
|
||||
}
|
||||
|
||||
|
||||
def list_pdf_plugins() -> list[str]:
|
||||
return list(_PDF_PLUGINS.keys())
|
||||
|
||||
|
||||
def list_docx_plugins() -> list[str]:
|
||||
return list(_DOCX_PLUGINS.keys())
|
||||
|
||||
|
||||
def list_ocr_plugins() -> list[str]:
|
||||
return list(_OCR_PLUGINS.keys())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
# 进程级插件单例缓存
|
||||
_pdf_plugin_cache: dict[str, PdfTextExtractor] = {}
|
||||
_docx_plugin_cache: dict[str, DocxParser] = {}
|
||||
_ocr_plugin_cache: dict[str, OcrEngine] = {}
|
||||
|
||||
|
||||
def _get_pdf_plugin() -> PdfTextExtractor:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.pdf.plugin or "pypdf"
|
||||
if name in _pdf_plugin_cache:
|
||||
return _pdf_plugin_cache[name]
|
||||
cls = _PDF_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 PDF 插件,降级到 pypdf", requested=name)
|
||||
cls = PypdfTextExtractor
|
||||
name = "pypdf"
|
||||
instance = cls()
|
||||
_pdf_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def _get_docx_plugin() -> DocxParser:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.docx.plugin or "python_docx"
|
||||
if name in _docx_plugin_cache:
|
||||
return _docx_plugin_cache[name]
|
||||
cls = _DOCX_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 DOCX 插件,降级到 python_docx", requested=name)
|
||||
cls = PythonDocxParser
|
||||
name = "python_docx"
|
||||
instance = cls()
|
||||
_docx_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def _get_ocr_plugin() -> OcrEngine:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.ocr.plugin or "rapidocr"
|
||||
if name in _ocr_plugin_cache:
|
||||
return _ocr_plugin_cache[name]
|
||||
cls = _OCR_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 OCR 插件,降级到 rapidocr", requested=name)
|
||||
cls = RapidocrOcrEngine
|
||||
name = "rapidocr"
|
||||
instance = cls()
|
||||
_ocr_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def invalidate_parser_plugin_cache(kind: str | None = None) -> None:
|
||||
"""清除插件缓存(runtime_settings 更新后调用)"""
|
||||
if kind is None or kind == "pdf":
|
||||
_pdf_plugin_cache.clear()
|
||||
if kind is None or kind == "docx":
|
||||
_docx_plugin_cache.clear()
|
||||
if kind is None or kind == "ocr":
|
||||
_ocr_plugin_cache.clear()
|
||||
# 重置 RapidocrOcrEngine 类级标记,允许重新初始化
|
||||
RapidocrOcrEngine._engine = None
|
||||
RapidocrOcrEngine._unavailable = False
|
||||
TesseractOcrEngine._unavailable = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 顶层解析函数
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _parse_pdf(content: bytes) -> str:
|
||||
"""PDF 解析:优先文本层插件;文本层为空时降级 OCR 插件"""
|
||||
rt = get_runtime_settings()
|
||||
pdf_plugin = _get_pdf_plugin()
|
||||
try:
|
||||
text_layer = pdf_plugin.extract_text(content)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"PDF 文本层提取失败,尝试 OCR 降级",
|
||||
plugin=rt.parsers.pdf.plugin,
|
||||
error=str(exc),
|
||||
exc_info=True,
|
||||
)
|
||||
text_layer = ""
|
||||
|
||||
# 文本层非空:直接返回
|
||||
if text_layer:
|
||||
return text_layer
|
||||
|
||||
# 文本层为空 → 尝试 OCR 降级
|
||||
# 文本层为空 → OCR 降级
|
||||
if not settings.pdf_ocr_enabled:
|
||||
return ""
|
||||
|
||||
ocr_text = _ocr_pdf(
|
||||
content, max_pages=settings.pdf_ocr_max_pages, dpi=settings.pdf_ocr_dpi
|
||||
ocr_plugin = _get_ocr_plugin()
|
||||
return ocr_plugin.ocr_pdf(
|
||||
content,
|
||||
max_pages=rt.parsers.ocr.params.get("max_pages", settings.pdf_ocr_max_pages),
|
||||
dpi=rt.parsers.ocr.params.get("dpi", settings.pdf_ocr_dpi),
|
||||
)
|
||||
return ocr_text
|
||||
|
||||
|
||||
def _ocr_pdf(content: bytes, max_pages: int, dpi: int) -> str:
|
||||
"""对扫描件 PDF 跑 OCR:渲染每页 → 识别 → 拼接
|
||||
|
||||
返回空字符串的场景:依赖未安装 / 渲染或识别异常 / 无识别结果。
|
||||
任何异常仅告警不抛出,由上游按"无法提取文本"处理。
|
||||
"""
|
||||
global _ocr_engine, _ocr_unavailable
|
||||
|
||||
if _ocr_unavailable:
|
||||
return ""
|
||||
|
||||
# 1. 懒加载 OCR 引擎
|
||||
if _ocr_engine is None:
|
||||
try:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
_ocr_engine = RapidOCR()
|
||||
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
|
||||
except Exception:
|
||||
_ocr_unavailable = True
|
||||
logger.warning("OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True)
|
||||
return ""
|
||||
|
||||
# 2. 渲染并识别
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
result, _ = _ocr_engine(pil_image)
|
||||
if result:
|
||||
# result: [[box, text, score], ...],按行拼接
|
||||
lines = [
|
||||
item[1] for item in result if item and len(item) >= 2 and item[1]
|
||||
]
|
||||
if lines:
|
||||
page_texts.append("\n".join(lines))
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning("PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True)
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_docx(content: bytes) -> str:
|
||||
"""DOCX 段落文本拼接(不含表格/页眉页脚)"""
|
||||
from docx import Document # type: ignore[import-untyped]
|
||||
|
||||
document = Document(io.BytesIO(content))
|
||||
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
|
||||
return "\n".join(parts).strip()
|
||||
"""DOCX 解析"""
|
||||
return _get_docx_plugin().extract_text(content)
|
||||
|
||||
|
||||
# 扩展名 → 解析函数映射(启动时构建,避免每次请求重复构造)
|
||||
# 扩展名 → 解析函数映射
|
||||
_PARSERS: dict[str, Callable[[bytes], str]] = {
|
||||
".txt": _parse_text,
|
||||
".md": _parse_text,
|
||||
|
||||
+13
-37
@@ -6,10 +6,12 @@
|
||||
内存注册表为主(记录 status/created_at/updated_at/result/error),
|
||||
Redis 为持久镜像(key: ingest_task:{task_id}),每次状态迁移同步写入;
|
||||
Redis 不可用或写入失败仅记录 warning,不影响任务执行。
|
||||
|
||||
文本去重委托给 app.core.dedup.get_dedup_strategy(),按 runtime_settings.dedup
|
||||
选择策略(none/sha256/simhash)。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
@@ -18,6 +20,7 @@ from typing import Any
|
||||
import structlog
|
||||
|
||||
from app.config import Settings
|
||||
from app.core.dedup import DEDUP_KEY_PREFIX, get_dedup_strategy
|
||||
from app.core.ingestion import Ingester, IngestionError
|
||||
from app.models.document import DocumentInput
|
||||
from app.services.redis import RedisCache
|
||||
@@ -26,8 +29,6 @@ logger = structlog.get_logger()
|
||||
|
||||
# Redis 任务状态 key 前缀
|
||||
REDIS_KEY_PREFIX = "ingest_task:"
|
||||
# Redis 文本去重 key 前缀(value 为已入库文档的 IngestionResult JSON)
|
||||
DEDUP_KEY_PREFIX = "dedup:sha256:"
|
||||
|
||||
|
||||
class IngestTaskStatus(StrEnum):
|
||||
@@ -71,17 +72,17 @@ class IngestTaskManager:
|
||||
async def submit(self, doc: DocumentInput) -> str:
|
||||
"""登记入库任务并后台执行,立即返回 task_id
|
||||
|
||||
文本去重:基于 doc.text 的 sha256 在 Redis 中查重;命中则直接复用旧
|
||||
IngestionResult(仅置 deduplicated=True),不重跑流水线;未命中走原
|
||||
异步入库流程,完成后写入去重记录供后续命中复用。Redis 不可用时跳过
|
||||
去重,按原流程执行,不影响主流程。
|
||||
文本去重:按 runtime_settings.dedup 选择策略(none/sha256/simhash);
|
||||
命中则直接复用旧 IngestionResult(仅置 deduplicated=True),不重跑流水线;
|
||||
未命中走原异步入库流程,完成后写入去重记录供后续命中复用。
|
||||
Redis 不可用或策略=none 时跳过,按原流程执行。
|
||||
"""
|
||||
task_id = uuid.uuid4().hex
|
||||
now = _utc_now_iso()
|
||||
text_hash = hashlib.sha256(doc.text.encode("utf-8")).hexdigest()
|
||||
dedup = get_dedup_strategy(self._redis)
|
||||
|
||||
# 1. 去重命中:直接置 done,复用旧结果,不调 _run
|
||||
dedup_record = await self._lookup_dedup(text_hash)
|
||||
dedup_record = await dedup.lookup(doc.text)
|
||||
if dedup_record is not None:
|
||||
result_dict = dict(dedup_record)
|
||||
result_dict["deduplicated"] = True
|
||||
@@ -111,7 +112,7 @@ class IngestTaskManager:
|
||||
"error": None,
|
||||
}
|
||||
self._schedule_mirror(task_id)
|
||||
background = asyncio.create_task(self._run(task_id, doc, text_hash))
|
||||
background = asyncio.create_task(self._run(task_id, doc, dedup))
|
||||
self._background_tasks.add(background)
|
||||
background.add_done_callback(self._background_tasks.discard)
|
||||
logger.info("入库任务已登记", task_id=task_id, title=doc.title)
|
||||
@@ -151,7 +152,7 @@ class IngestTaskManager:
|
||||
raise TimeoutError(f"入库任务 {task_id} 在 {timeout}s 内未进入终态")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def _run(self, task_id: str, doc: DocumentInput, text_hash: str) -> None:
|
||||
async def _run(self, task_id: str, doc: DocumentInput, dedup: Any) -> None:
|
||||
"""后台执行入库:并发限流 + 阶段状态推进 + 结果/错误落账 + 去重记录写入"""
|
||||
async with self._semaphore:
|
||||
try:
|
||||
@@ -185,34 +186,9 @@ class IngestTaskManager:
|
||||
result=result_dict,
|
||||
)
|
||||
self._schedule_mirror(task_id)
|
||||
await self._record_dedup(text_hash, result_dict)
|
||||
await dedup.record(doc.text, result_dict)
|
||||
logger.info("入库任务完成", task_id=task_id)
|
||||
|
||||
async def _lookup_dedup(self, text_hash: str) -> dict[str, Any] | None:
|
||||
"""查询文本去重记录;Redis 不可用或异常时降级为未命中"""
|
||||
if self._redis is None:
|
||||
return None
|
||||
try:
|
||||
return await self._redis.get_json(f"{DEDUP_KEY_PREFIX}{text_hash}")
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"去重记录查询失败,降级为未命中", text_hash=text_hash, exc_info=True
|
||||
)
|
||||
return None
|
||||
|
||||
async def _record_dedup(self, text_hash: str, result_dict: dict[str, Any]) -> None:
|
||||
"""写入文本去重记录(含完整 IngestionResult),供后续命中复用;失败仅告警"""
|
||||
if self._redis is None:
|
||||
return
|
||||
try:
|
||||
await self._redis.set_json(
|
||||
f"{DEDUP_KEY_PREFIX}{text_hash}",
|
||||
result_dict,
|
||||
ttl=self._settings.ingest_task_ttl_done,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("去重记录写入失败", text_hash=text_hash, exc_info=True)
|
||||
|
||||
def _finish_failed(self, task_id: str, error: dict[str, Any]) -> None:
|
||||
"""将任务置为 failed 并记录错误信息"""
|
||||
self._tasks[task_id].update(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""query 解析与分类路由模块
|
||||
|
||||
分层 RAG 在线侧第一步:用 Ollama 小模型将用户 query 解析为结构化 JSON
|
||||
分层 RAG 在线侧第一步:用 LLM(Ollama 或 OpenAI 兼容服务,由
|
||||
runtime_settings.models.query 决定)将用户 query 解析为结构化 JSON
|
||||
(命中类目+置信度、rewrite 后 query、关键词),再由纯函数做路由决策:
|
||||
- 高置信且命中类目数 <= 上限 → 按类目过滤检索
|
||||
- 低置信 / 解析失败 / 命中类目过多 → 全库兜底(不丢召回)
|
||||
@@ -16,8 +17,8 @@ import structlog
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
from app.config import settings
|
||||
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory, load_taxonomy
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
from app.services.redis import RedisCache, get_cache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
@@ -106,17 +107,22 @@ def decide_route(parsed: ParsedQuery, threshold: float, max_categories: int) ->
|
||||
|
||||
|
||||
class QueryParser:
|
||||
"""query 解析器:调用 Ollama 小模型将 query 解析为结构化 JSON"""
|
||||
"""query 解析器:调用 LLM 将 query 解析为结构化 JSON"""
|
||||
|
||||
def __init__(
|
||||
self, ollama: OllamaClient, taxonomy: list[TaxonomyCategory], cache: RedisCache | None = None
|
||||
self,
|
||||
ollama: LLMClient | None = None,
|
||||
taxonomy: list[TaxonomyCategory] | None = None,
|
||||
cache: RedisCache | None = None,
|
||||
) -> None:
|
||||
self.ollama = ollama
|
||||
self.taxonomy = taxonomy
|
||||
# 默认按 runtime_settings.models.query 选择 LLM 实现;
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("query")
|
||||
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
|
||||
# 解析结果缓存,缺省用全局单例;RedisCache 全操作容错,缓存不可用时退化为无缓存行为
|
||||
self.cache = cache if cache is not None else get_cache()
|
||||
# 可作为路由命中类目的名字集合(uncategorized 不可作为路由命中类目)
|
||||
self._routable_names = {c.name for c in taxonomy if c.name != UNCATEGORIZED}
|
||||
self._routable_names = {c.name for c in self.taxonomy if c.name != UNCATEGORIZED}
|
||||
|
||||
async def parse(self, query: str) -> ParsedQuery:
|
||||
"""调用 LLM 将 query 解析为结构化结果
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""检索结果 AI 总结
|
||||
|
||||
对检索返回的 chunk 命中结果,调用 Ollama 本地模型生成一段针对用户 query 的总结回答。
|
||||
对检索返回的 chunk 命中结果,调用 LLM(Ollama 或 OpenAI 兼容服务,由
|
||||
runtime_settings.models.query 决定)生成一段针对用户 query 的总结回答。
|
||||
仅基于检索结果内容,不编造未提及的信息。
|
||||
"""
|
||||
|
||||
@@ -8,7 +9,7 @@ import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.models.search import SearchHit
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
@@ -16,8 +17,10 @@ logger = structlog.get_logger()
|
||||
class ResultSummarizer:
|
||||
"""检索结果总结器"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
def __init__(self, ollama: LLMClient | None = None) -> None:
|
||||
# 默认按 runtime_settings.models.query 选择 LLM 实现;
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("query")
|
||||
|
||||
async def summarize(self, query: str, hits: list[SearchHit]) -> str:
|
||||
"""对检索结果生成针对 query 的总结
|
||||
|
||||
@@ -21,7 +21,7 @@ from app.core.result_summarizer import ResultSummarizer
|
||||
from app.core.sparse import SparseEncoder
|
||||
from app.models.knowledge import load_taxonomy
|
||||
from app.models.search import ExtractedInfo, SearchHit, SearchRequest, SearchResponse
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import create_llm_client
|
||||
from app.services.qdrant import (
|
||||
COLLECTION_CHUNKS,
|
||||
COLLECTION_L1,
|
||||
@@ -52,7 +52,7 @@ class Retriever:
|
||||
) -> None:
|
||||
self.qdrant = qdrant or QdrantService()
|
||||
self.query_parser = query_parser or QueryParser(
|
||||
ollama=OllamaClient(),
|
||||
ollama=create_llm_client("query"),
|
||||
taxonomy=load_taxonomy(settings.taxonomy_path),
|
||||
)
|
||||
self.embedding = embedding or create_embedding_service()
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""运行时可调配置(runtime settings)
|
||||
|
||||
与 `app.config.settings`(启动时从环境变量加载、不可变)互补:本模块管理运行时
|
||||
可通过管理后台动态调整的子集,持久化到 JSON 文件,启动时加载覆盖到内存单例。
|
||||
|
||||
设计要点:
|
||||
- RuntimeSettings 只包含「可运行时调整」的字段(模型/解析插件/去重等),
|
||||
不包含敏感或启动期固定的字段(端口、数据库连接等)。
|
||||
- 持久化路径默认 `data/runtime_settings.json`,可由 env `RUNTIME_SETTINGS_PATH` 覆盖。
|
||||
- 加载失败/文件缺失时回退到默认值,不阻塞启动。
|
||||
- 写入采用「先临时文件后 rename」原子替换,避免半写损坏。
|
||||
- 全模块只通过 `get_runtime_settings()` 访问单例,避免直接读 JSON。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from typing import Any, Literal
|
||||
|
||||
import structlog
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 默认持久化路径(相对工作目录);可由 env 覆盖
|
||||
DEFAULT_RUNTIME_SETTINGS_PATH = "./data/runtime_settings.json"
|
||||
|
||||
_lock = RLock()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 模型配置(文档总结 / 查询 / 分类 各一份独立配置)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class LlmProviderConfig(BaseModel):
|
||||
"""单个用途的 LLM 提供方配置"""
|
||||
|
||||
provider: Literal["ollama", "openai_compatible"] = Field(
|
||||
default="ollama", description="提供方:ollama 走 Ollama HTTP;openai_compatible 走 OpenAI 兼容 chat/completions"
|
||||
)
|
||||
base_url: str = Field(default="", description="服务地址;空则 ollama 用 settings.ollama_base_url,openai_compatible 用 settings.openai_base_url")
|
||||
api_key: str = Field(default="", description="API Key(仅 openai_compatible 需要;ollama 忽略)")
|
||||
model: str = Field(default="", description="模型名;空则 ollama 用 settings.ollama_model,openai_compatible 用 settings.embedding_model 同级(如 gpt-4o-mini)")
|
||||
timeout: float = Field(default=120.0, description="请求超时秒数")
|
||||
temperature: float = Field(default=0.3, description="采样温度(0~2)")
|
||||
|
||||
|
||||
class ModelSettings(BaseModel):
|
||||
"""模型相关运行时配置:三种用途独立配置"""
|
||||
|
||||
summarize: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档三级总结")
|
||||
query: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="检索链路 query 解析与结果总结")
|
||||
classify: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档分类判定")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 解析插件配置
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class PluginConfig(BaseModel):
|
||||
"""单个解析插件的选择与参数"""
|
||||
|
||||
plugin: str = Field(default="", description="插件名;空则用默认")
|
||||
params: dict[str, Any] = Field(default_factory=dict, description="插件参数(透传给插件实现)")
|
||||
|
||||
|
||||
class ParserSettings(BaseModel):
|
||||
"""解析插件运行时配置"""
|
||||
|
||||
ocr: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="rapidocr"), description="OCR 插件")
|
||||
pdf: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="pypdf"), description="PDF 文本层提取插件")
|
||||
docx: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="python_docx"), description="DOCX 解析插件")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 去重策略配置
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class DedupSettings(BaseModel):
|
||||
"""文本去重策略运行时配置"""
|
||||
|
||||
strategy: Literal["none", "sha256", "simhash"] = Field(
|
||||
default="sha256", description="去重策略:none 关闭;sha256 精确匹配;simhash 近似匹配"
|
||||
)
|
||||
simhash_threshold: int = Field(default=3, description="simhash 海明距离阈值(仅 strategy=simhash 生效,0~64)")
|
||||
ttl_seconds: int = Field(default=86400, description="去重记录 Redis 保留秒数")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 顶层 RuntimeSettings
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class RuntimeSettings(BaseModel):
|
||||
"""运行时可调配置顶层模型"""
|
||||
|
||||
models: ModelSettings = Field(default_factory=ModelSettings)
|
||||
parsers: ParserSettings = Field(default_factory=ParserSettings)
|
||||
dedup: DedupSettings = Field(default_factory=DedupSettings)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 单例 + 持久化
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_runtime_settings: RuntimeSettings | None = None
|
||||
|
||||
|
||||
def _resolve_path() -> Path:
|
||||
"""解析持久化文件路径:env RUNTIME_SETTINGS_PATH > settings 自定义 > 默认"""
|
||||
path_str = os.environ.get("RUNTIME_SETTINGS_PATH", "") or getattr(settings, "runtime_settings_path", "") or DEFAULT_RUNTIME_SETTINGS_PATH
|
||||
return Path(path_str).expanduser().resolve()
|
||||
|
||||
|
||||
def _default_with_env_fallback() -> RuntimeSettings:
|
||||
"""构造默认 RuntimeSettings,并把启动 env 中已有的模型相关字段填充进去
|
||||
|
||||
这样首次启动(无持久化文件)时,UI 显示的不是空字符串而是 env 当前值。
|
||||
"""
|
||||
cfg = RuntimeSettings()
|
||||
# 模型默认值沿用 env
|
||||
cfg.models.summarize.base_url = settings.ollama_base_url
|
||||
cfg.models.summarize.model = settings.ollama_model
|
||||
cfg.models.query.base_url = settings.ollama_base_url
|
||||
cfg.models.query.model = settings.ollama_model
|
||||
cfg.models.classify.base_url = settings.ollama_base_url
|
||||
cfg.models.classify.model = settings.ollama_model
|
||||
# 若 env 提供了 OpenAI key/url,预填到 openai_compatible 字段方便切换
|
||||
if settings.openai_api_key:
|
||||
for usage in ("summarize", "query", "classify"):
|
||||
getattr(cfg.models, usage).api_key = settings.openai_api_key
|
||||
getattr(cfg.models, usage).base_url = settings.openai_base_url if settings.openai_base_url else getattr(cfg.models, usage).base_url
|
||||
return cfg
|
||||
|
||||
|
||||
def load_runtime_settings() -> RuntimeSettings:
|
||||
"""从磁盘加载 RuntimeSettings;文件缺失或损坏时回退到默认值(带 env 兜底)"""
|
||||
path = _resolve_path()
|
||||
try:
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return RuntimeSettings.model_validate(data)
|
||||
except Exception:
|
||||
logger.warning("RuntimeSettings 加载失败,回退默认值", path=str(path), exc_info=True)
|
||||
return _default_with_env_fallback()
|
||||
|
||||
|
||||
def save_runtime_settings(cfg: RuntimeSettings) -> None:
|
||||
"""原子写入 RuntimeSettings 到磁盘(先临时文件后 rename)"""
|
||||
path = _resolve_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = cfg.model_dump(mode="json")
|
||||
# 写到同目录临时文件再 rename,避免半写损坏
|
||||
fd, tmp_path = tempfile.mkstemp(prefix=".runtime_settings.", suffix=".tmp", dir=str(path.parent))
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp_path, path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def get_runtime_settings() -> RuntimeSettings:
|
||||
"""获取 RuntimeSettings 单例(首次调用时加载)"""
|
||||
global _runtime_settings
|
||||
with _lock:
|
||||
if _runtime_settings is None:
|
||||
_runtime_settings = load_runtime_settings()
|
||||
return _runtime_settings
|
||||
|
||||
|
||||
def update_runtime_settings(patch: dict[str, Any]) -> RuntimeSettings:
|
||||
"""以 patch 字典更新 RuntimeSettings 并持久化
|
||||
|
||||
支持部分更新(顶层键可缺失,缺省保留原值)。例如:
|
||||
update_runtime_settings({"models": {"summarize": {"model": "qwen2.5:3b"}}})
|
||||
"""
|
||||
with _lock:
|
||||
current = get_runtime_settings()
|
||||
merged = current.model_dump(mode="json")
|
||||
_deep_merge(merged, patch)
|
||||
new_cfg = RuntimeSettings.model_validate(merged)
|
||||
save_runtime_settings(new_cfg)
|
||||
# 替换单例后立即返回,新值对所有后续读取生效
|
||||
global _runtime_settings
|
||||
_runtime_settings = new_cfg
|
||||
logger.info("RuntimeSettings 已更新并持久化", path=str(_resolve_path()))
|
||||
return new_cfg
|
||||
|
||||
|
||||
def _deep_merge(target: dict[str, Any], patch: dict[str, Any]) -> None:
|
||||
"""递归把 patch 合并到 target(同 key 字典则递归,否则覆盖)"""
|
||||
for k, v in patch.items():
|
||||
if k in target and isinstance(target[k], dict) and isinstance(v, dict):
|
||||
_deep_merge(target[k], v)
|
||||
else:
|
||||
target[k] = v
|
||||
|
||||
|
||||
def reload_runtime_settings() -> RuntimeSettings:
|
||||
"""强制从磁盘重新加载(管理后台触发)"""
|
||||
with _lock:
|
||||
global _runtime_settings
|
||||
_runtime_settings = load_runtime_settings()
|
||||
return _runtime_settings
|
||||
|
||||
|
||||
def reset_runtime_settings() -> RuntimeSettings:
|
||||
"""重置为默认值并持久化(管理后台触发)"""
|
||||
with _lock:
|
||||
new_cfg = _default_with_env_fallback()
|
||||
save_runtime_settings(new_cfg)
|
||||
global _runtime_settings
|
||||
_runtime_settings = new_cfg
|
||||
return new_cfg
|
||||
@@ -1,6 +1,7 @@
|
||||
"""文档三级总结模块
|
||||
|
||||
通过 Ollama 本地小模型对文档进行分级总结:
|
||||
通过 LLM(Ollama 或 OpenAI 兼容服务,由 runtime_settings.models.summarize 决定)
|
||||
对文档进行分级总结:
|
||||
- L1: 总结(一句话高度概括)
|
||||
- L2: 大纲(主要章节和关键主题)
|
||||
- L3: 内容大纲(每个章节的详细内容摘要)
|
||||
@@ -12,7 +13,7 @@ import structlog
|
||||
|
||||
from app.core.headings import parse_headings, render_outline
|
||||
from app.models.document import DocumentSummary, SummaryLevel
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
@@ -23,8 +24,10 @@ MIN_TEXT_LENGTH_FOR_L3 = 500
|
||||
class Summarizer:
|
||||
"""文档三级总结器"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
def __init__(self, ollama: LLMClient | None = None) -> None:
|
||||
# 默认按 runtime_settings 选择 LLM 实现(Ollama 或 OpenAI 兼容);
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("summarize")
|
||||
|
||||
async def summarize(self, text: str, *, title: str = "") -> DocumentSummary:
|
||||
"""对文档文本进行三级总结
|
||||
|
||||
Reference in New Issue
Block a user