feat: 完成全量功能开发,包括前端管理后台与后端服务优化

此提交实现了完整的知识库管理系统:
1. 新增Vue3 + Antd Vue前端管理后台,包含登录、文档管理、检索、类目设置等完整页面
2. 重构后端LLM调用抽象层,支持Ollama与OpenAI兼容服务动态切换
3. 调整默认嵌入模型配置为本地bge-m3模式
4. 优化入库任务去重逻辑与缓存清理机制
5. 完善Docker镜像构建与docker-compose部署配置
6. 修复多项测试用例与兼容性问题
7. 新增运行时配置API,支持动态调整系统参数
This commit is contained in:
2026-07-31 12:05:25 +08:00
parent fdb664e546
commit 2ab8b56a01
52 changed files with 7030 additions and 171 deletions
+114
View File
@@ -0,0 +1,114 @@
"""运行时配置 APIGET/PUT /api/v1/settings、GET /api/v1/settings/schema、POST /api/v1/settings/reset
GET 任何登录用户可读;PUT/reset 需 admin。PUT 后会清空 LLM 客户端 / 解析插件 /
去重策略三处进程级缓存,使新配置立即对后续请求生效。
"""
from typing import Any
import structlog
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from app.api.response import ApiError, ok
from app.core.auth import AuthUser, get_current_user, require_admin
from app.core.dedup import invalidate_dedup_strategy_cache
from app.core.file_parser import (
list_docx_plugins,
list_ocr_plugins,
list_pdf_plugins,
invalidate_parser_plugin_cache,
)
from app.core.runtime_settings import (
RuntimeSettings,
get_runtime_settings,
reset_runtime_settings,
update_runtime_settings,
)
from app.services.llm import invalidate_llm_client_cache
logger = structlog.get_logger()
router = APIRouter(prefix="/api/v1", tags=["settings"])
class SettingsUpdateRequest(BaseModel):
"""Settings PATCH body:任意子树可缺省,缺省字段保留原值
例:{"models": {"summarize": {"model": "qwen2.5:3b"}}}、
{"parsers": {"ocr": {"plugin": "tesseract"}}}、
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
"""
models: dict[str, Any] | None = None
parsers: dict[str, Any] | None = None
dedup: dict[str, Any] | None = None
@router.get("/settings")
async def get_settings(user: AuthUser = Depends(get_current_user)) -> dict[str, Any]:
"""返回当前 RuntimeSettings(任何登录用户可读)"""
cfg = get_runtime_settings()
return ok(cfg.model_dump(mode="json"))
@router.put("/settings")
async def update_settings(
body: SettingsUpdateRequest,
user: AuthUser = Depends(require_admin),
) -> dict[str, Any]:
"""部分更新 RuntimeSettings(仅 admin
更新成功后清空 LLM 客户端 / 解析插件 / 去重策略三处进程级缓存,
使新配置立即对后续请求生效。
"""
patch = body.model_dump(exclude_none=True)
if not patch:
raise ApiError(1001, "请求体为空,未提供任何待更新字段")
try:
cfg = update_runtime_settings(patch)
except Exception as exc:
logger.error("RuntimeSettings 更新失败", error=str(exc), exc_info=True)
raise ApiError(2000, f"配置更新失败: {exc}") from exc
# 清缓存:让后续读取拿到新配置
invalidate_llm_client_cache()
invalidate_parser_plugin_cache()
invalidate_dedup_strategy_cache()
logger.info("RuntimeSettings 已更新并清理缓存", operator=user.username)
return ok(cfg.model_dump(mode="json"))
@router.get("/settings/schema")
async def get_settings_schema(
user: AuthUser = Depends(get_current_user),
) -> dict[str, Any]:
"""返回可选插件与策略列表(前端 Settings 页渲染选项用)"""
return ok(
{
"llm_providers": ["ollama", "openai_compatible"],
"pdf_plugins": list_pdf_plugins(),
"docx_plugins": list_docx_plugins(),
"ocr_plugins": list_ocr_plugins(),
"dedup_strategies": ["none", "sha256", "simhash"],
}
)
@router.post("/settings/reset")
async def reset_settings(
user: AuthUser = Depends(require_admin),
) -> dict[str, Any]:
"""重置 RuntimeSettings 为默认值(仅 admin),同时清缓存"""
try:
cfg = reset_runtime_settings()
except Exception as exc:
logger.error("RuntimeSettings 重置失败", error=str(exc), exc_info=True)
raise ApiError(2000, f"配置重置失败: {exc}") from exc
invalidate_llm_client_cache()
invalidate_parser_plugin_cache()
invalidate_dedup_strategy_cache()
logger.info("RuntimeSettings 已重置为默认值", operator=user.username)
return ok(cfg.model_dump(mode="json"))
+3 -2
View File
@@ -9,11 +9,12 @@ class Settings(BaseSettings):
log_level: str = "info"
# 嵌入模型
embedding_provider: str = "openai" # openai | local
embedding_provider: str = "local" # openai | local
openai_api_key: str = ""
openai_base_url: str = "https://api.openai.com/v1"
embedding_model: str = "text-embedding-3-small"
embedding_dimension: int = 1536
# bge-m3(本地 Ollama 嵌入)维度为 1024;切换 openai provider 时需同步改为 1536
embedding_dimension: int = 1024
# Ollama 本地模型(用于文档三级总结)
ollama_base_url: str = "http://localhost:11434"
+7 -4
View File
@@ -1,6 +1,7 @@
"""文档分类器
入库链路第二步:基于 L1 总结,用 Ollama 小模型将文档判定为 taxonomy 中的
入库链路第二步:基于 L1 总结,用 LLMOllama 或 OpenAI 兼容服务,由
runtime_settings.models.classify 决定)将文档判定为 taxonomy 中的
主类目 + 附加标签:
- LLM 输出解析失败 / 类目名不在 taxonomy → 归 uncategorizedconfidence=0.0
- 置信度低于阈值 → 主类目归 uncategorized,候选类目名保留进 tags(软召回用)
@@ -13,7 +14,7 @@ import structlog
from app.config import settings
from app.models.knowledge import UNCATEGORIZED, CategoryResult, TaxonomyCategory, load_taxonomy
from app.services.ollama import OllamaClient
from app.services.llm import LLMClient, create_llm_client
logger = structlog.get_logger()
@@ -47,8 +48,10 @@ def _extract_json(raw: str) -> dict | None:
class Classifier:
"""文档分类器:将 L1 总结判定为 taxonomy 主类目 + 附加标签"""
def __init__(self, ollama: OllamaClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
self.ollama = ollama or OllamaClient()
def __init__(self, ollama: LLMClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
# 默认按 runtime_settings.models.classify 选择 LLM 实现;
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("classify")
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
# 合法类目名集合(含 uncategorized
self._valid_names = {c.name for c in self.taxonomy}
+216
View File
@@ -0,0 +1,216 @@
"""文本去重策略工厂
支持三种策略:
- none:关闭去重,每次都跑完整流水线
- sha256:精确匹配(hash 完全相同才算重复)
- simhash:近似匹配(海明距离 <= 阈值视为重复)
每种策略实现 DedupStrategy 协议:lookup(text) -> dict|None、record(text, result) -> None。
工厂 get_dedup_strategy() 根据 runtime_settings.dedup 选择实现。
所有策略共享同一 Redis key 前缀,但 key 后缀按策略区分避免冲突。
"""
from __future__ import annotations
import hashlib
from typing import Any, Protocol, runtime_checkable
import structlog
from app.core.runtime_settings import get_runtime_settings
from app.services.redis import RedisCache
logger = structlog.get_logger()
# Redis 去重 key 前缀,按策略分桶
_DEDUP_KEY_PREFIX = "dedup:"
@runtime_checkable
class DedupStrategy(Protocol):
"""去重策略协议"""
async def lookup(self, text: str) -> dict[str, Any] | None:
"""查询文本是否已入库;命中返回旧 IngestionResultdict),未命中返回 None"""
...
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
"""记录文本已入库,供后续命中复用"""
...
class NoopDedupStrategy:
"""关闭去重:永远不命中,也不记录"""
async def lookup(self, text: str) -> dict[str, Any] | None:
return None
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
return None
class Sha256DedupStrategy:
"""SHA256 精确去重:完全相同文本才算重复"""
def __init__(self, redis: RedisCache, ttl_seconds: int) -> None:
self._redis = redis
self._ttl = ttl_seconds
def _key(self, text: str) -> str:
h = hashlib.sha256(text.encode("utf-8")).hexdigest()
return f"{_DEDUP_KEY_PREFIX}sha256:{h}"
async def lookup(self, text: str) -> dict[str, Any] | None:
try:
return await self._redis.get_json(self._key(text))
except Exception:
logger.warning("sha256 去重查询失败,降级未命中", exc_info=True)
return None
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
try:
await self._redis.set_json(self._key(text), result_dict, ttl=self._ttl)
except Exception:
logger.warning("sha256 去重记录写入失败", exc_info=True)
# SimHash 实现(64 位)
_MASK_64 = (1 << 64) - 1
def _simhash(text: str, token_size: int = 4) -> int:
"""计算文本 64 位 simhash
简化实现:按 token_size 字符滑窗分词,每段 md5 → 128 位 → 取低 64 位作为 hash,
逐位加权(hash 该位为 1 则 +1,为 0 则 -1),最终符号位定 1/0。
"""
if not text:
return 0
tokens = [text[i : i + token_size] for i in range(0, len(text), token_size)]
weights = [1] * 64
vec = [0] * 64
for token in tokens:
h = int(hashlib.md5(token.encode("utf-8")).hexdigest(), 16) & _MASK_64
for i in range(64):
bit = (h >> i) & 1
vec[i] += weights[i] if bit else -weights[i]
fingerprint = 0
for i in range(64):
if vec[i] > 0:
fingerprint |= (1 << i)
return fingerprint
def _hamming_distance(a: int, b: int) -> int:
return bin((a ^ b) & _MASK_64).count("1")
class SimhashDedupStrategy:
"""SimHash 近似去重:海明距离 <= 阈值视为重复
Redis 中存所有已入库文本的 simhashvalue 为 IngestionResult + simhash)。
lookup 时遍历所有候选 simhash 算海明距离,找最近的一个 <= 阈值则命中。
注:本实现为简化版,全表扫描,适合中小规模知识库;超大规模需换 LSH 索引。
"""
INDEX_KEY = f"{_DEDUP_KEY_PREFIX}simhash:index" # list 形式存所有 simhash+key
def __init__(self, redis: RedisCache, ttl_seconds: int, threshold: int = 3) -> None:
self._redis = redis
self._ttl = ttl_seconds
self._threshold = max(0, min(64, threshold))
async def lookup(self, text: str) -> dict[str, Any] | None:
try:
fingerprint = _simhash(text)
# 简化:扫描所有 simhash 记录找最近的
# 用 set:dedup:simhash:fps 存所有 fingerprint,每个 fp 对应一个 record key
# 这里用 list key 简化(适合小规模)
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
entries = index.get("entries", [])
best_dist = self._threshold + 1
best_key: str | None = None
for entry in entries:
fp = entry.get("fingerprint")
key = entry.get("key")
if fp is None or key is None:
continue
dist = _hamming_distance(fingerprint, int(fp))
if dist <= self._threshold and dist < best_dist:
best_dist = dist
best_key = key
if best_key is None:
return None
return await self._redis.get_json(best_key)
except Exception:
logger.warning("simhash 去重查询失败,降级未命中", exc_info=True)
return None
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
try:
fingerprint = _simhash(text)
key = f"{_DEDUP_KEY_PREFIX}simhash:{fingerprint:016x}"
# 1. 写记录
await self._redis.set_json(key, result_dict, ttl=self._ttl)
# 2. 更新索引
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
entries = index.get("entries", [])
entries.append({"fingerprint": fingerprint, "key": key})
await self._redis.set_json(self.INDEX_KEY, {"entries": entries}, ttl=self._ttl)
except Exception:
logger.warning("simhash 去重记录写入失败", exc_info=True)
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
_strategy_cache: DedupStrategy | None = None
_strategy_signature: tuple[str, int, int] | None = None # (strategy, ttl, threshold)
def get_dedup_strategy(redis: RedisCache | None) -> DedupStrategy:
"""按 runtime_settings.dedup 构造去重策略
Redis 不可用时降级为 NoopDedupStrategy(关闭去重),不影响主流程。
策略配置变更时自动重建单例。
"""
global _strategy_cache, _strategy_signature
rt = get_runtime_settings()
sig = (rt.dedup.strategy, rt.dedup.ttl_seconds, rt.dedup.simhash_threshold)
if _strategy_cache is not None and _strategy_signature == sig:
return _strategy_cache
if redis is None or rt.dedup.strategy == "none":
strategy: DedupStrategy = NoopDedupStrategy()
elif rt.dedup.strategy == "sha256":
strategy = Sha256DedupStrategy(redis=redis, ttl_seconds=rt.dedup.ttl_seconds)
elif rt.dedup.strategy == "simhash":
strategy = SimhashDedupStrategy(
redis=redis, ttl_seconds=rt.dedup.ttl_seconds, threshold=rt.dedup.simhash_threshold
)
else:
logger.warning("未知去重策略,降级为 noop", requested=rt.dedup.strategy)
strategy = NoopDedupStrategy()
_strategy_cache = strategy
_strategy_signature = sig
return strategy
def invalidate_dedup_strategy_cache() -> None:
"""清除策略缓存(runtime_settings 更新后调用)"""
global _strategy_cache, _strategy_signature
_strategy_cache = None
_strategy_signature = None
def compute_text_hash(text: str) -> str:
"""计算文本哈希(兼容旧接口,sha256 策略下与 dedup key 一致)"""
return hashlib.sha256(text.encode("utf-8")).hexdigest()
# 兼容旧测试与外部引用:保留 DEDUP_KEY_PREFIX 导出
DEDUP_KEY_PREFIX = _DEDUP_KEY_PREFIX
+346 -87
View File
@@ -1,14 +1,19 @@
"""多格式文件文本提取:按扩展名分发到对应解析器
"""多格式文件文本提取:按扩展名分发到对应解析器(插件化)
支持的扩展名:
- .txt / .mdUTF-8 解码(errors="replace" 兜底)
- .html / .htm:标准库 html.parser 剥离标签提取可见文本
- .pdfpypdf 逐页 extract_text 拼接;文本层为空(扫描件/图片型)时
自动降级为 OCRpypdfium2 渲染 + rapidocr-onnxruntime 识别
- .docxpython-docx 段落文本拼接(不含表格/页眉页脚)
- .pdfPDF 文本层提取插件(默认 pypdf);文本层为空(扫描件)时降级到 OCR 插件
- .docxDOCX 解析插件(默认 python-docx
插件化设计:
- PdfTextExtractor / DocxParser / OcrEngine 三个 Protocol
- 每种插件类型有注册表 + 默认实现 + 工厂
- 工厂根据 runtime_settings.parsers 选择具体插件
- 未注册或依赖缺失时降级到默认插件并告警
未识别扩展名抛 ValueError("不支持的文件类型: {ext}")
解析异常统一包装为 ValueError("文件解析失败: {detail}"),原异常链式保留
解析异常统一包装为 ValueError("文件解析失败: {detail}")。
"""
from __future__ import annotations
@@ -17,17 +22,19 @@ import io
from collections.abc import Callable
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from typing import Any, Protocol, runtime_checkable
import structlog
from app.config import settings
from app.core.runtime_settings import get_runtime_settings
logger = structlog.get_logger()
# 模块级懒加载 OCR 引擎单例(首次调用时初始化,避免无扫描件场景白白下载模型)
_ocr_engine: Any | None = None
_ocr_unavailable: bool = False # 标记 OCR 依赖不可用,后续直接跳过避免重复尝试
# ---------------------------------------------------------------------------- #
# HTML / Text 解析(无插件化需求,保留原实现)
# ---------------------------------------------------------------------------- #
class _VisibleTextExtractor(HTMLParser):
@@ -51,13 +58,11 @@ class _VisibleTextExtractor(HTMLParser):
self._parts.append(data)
def get_text(self) -> str:
# 块级标签间用空格连接,再折叠多余空白
text = " ".join(self._parts)
return " ".join(text.split())
def _decode_html(content: bytes) -> str:
"""HTML 内容解码:优先 utf-8(带 BOM),失败回退 latin-1"""
try:
return content.decode("utf-8-sig")
except UnicodeDecodeError:
@@ -65,108 +70,362 @@ def _decode_html(content: bytes) -> str:
def _parse_text(content: bytes) -> str:
"""UTF-8 解码(errors=replace 兜底),保留原字符"""
return content.decode("utf-8", errors="replace")
def _parse_html(content: bytes) -> str:
"""HTML 剥离标签,保留可见文本"""
parser = _VisibleTextExtractor()
parser.feed(_decode_html(content))
parser.close()
return parser.get_text()
def _parse_pdf(content: bytes) -> str:
"""PDF 解析:优先 pypdf extract_text;文本层为空(扫描件)时降级 OCR
# ---------------------------------------------------------------------------- #
# 插件协议
# ---------------------------------------------------------------------------- #
OCR 流程:pypdfium2 渲染每页为 PIL Image → rapidocr-onnxruntime 识别 →
拼接每页识别出的文本。受 settings.pdf_ocr_* 控制:开关、最大页数、DPI。
OCR 依赖未安装或运行异常时降级返回空字符串(由上游 upload 端点拒绝入库)。
@runtime_checkable
class PdfTextExtractor(Protocol):
"""PDF 文本层提取插件协议"""
def extract_text(self, content: bytes) -> str:
"""从 PDF 二进制内容提取文本层;无文本层返回空字符串"""
...
@runtime_checkable
class DocxParser(Protocol):
"""DOCX 解析插件协议"""
def extract_text(self, content: bytes) -> str:
"""从 DOCX 二进制内容提取段落文本"""
...
@runtime_checkable
class OcrEngine(Protocol):
"""OCR 引擎插件协议"""
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
"""对 PDF 跑 OCR,返回识别文本;失败返回空字符串"""
...
# ---------------------------------------------------------------------------- #
# 默认 PDF 插件:pypdf
# ---------------------------------------------------------------------------- #
class PypdfTextExtractor:
"""pypdf 文本层提取(默认)"""
def extract_text(self, content: bytes) -> str:
from pypdf import PdfReader
reader = PdfReader(io.BytesIO(content))
parts: list[str] = []
for page in reader.pages:
text = page.extract_text() or ""
if text:
parts.append(text)
return "\n".join(parts).strip()
# ---------------------------------------------------------------------------- #
# 备选 PDF 插件:pdfplumberlazy import,依赖缺失时不可用)
# ---------------------------------------------------------------------------- #
class PdfplumberTextExtractor:
"""pdfplumber 文本层提取(备选,对复杂排版更友好)"""
def extract_text(self, content: bytes) -> str:
import pdfplumber # type: ignore[import-untyped]
parts: list[str] = []
with pdfplumber.open(io.BytesIO(content)) as pdf:
for page in pdf.pages:
text = page.extract_text() or ""
if text:
parts.append(text)
return "\n".join(parts).strip()
# ---------------------------------------------------------------------------- #
# 默认 DOCX 插件:python-docx
# ---------------------------------------------------------------------------- #
class PythonDocxParser:
"""python-docx 段落提取(默认)"""
def extract_text(self, content: bytes) -> str:
from docx import Document # type: ignore[import-untyped]
document = Document(io.BytesIO(content))
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
return "\n".join(parts).strip()
# ---------------------------------------------------------------------------- #
# 默认 OCR 插件:rapidocr-onnxruntime + pypdfium2
# ---------------------------------------------------------------------------- #
class RapidocrOcrEngine:
"""rapidocr-onnxruntime OCR 引擎(默认)
流程:pypdfium2 渲染每页为 PIL Image → rapidocr 识别 → 拼接。
类级懒加载单例(_engine/_unavailable 为类属性,跨实例共享),
依赖缺失时降级返回空文本。
"""
from pypdf import PdfReader
reader = PdfReader(io.BytesIO(content))
parts: list[str] = []
for page in reader.pages:
text = page.extract_text() or ""
if text:
parts.append(text)
text_layer = "\n".join(parts).strip()
_engine: Any | None = None
_unavailable: bool = False
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
if self._unavailable:
return ""
# 1. 懒加载 OCR 引擎(类级单例,跨实例共享)
if self._engine is None:
try:
from rapidocr_onnxruntime import RapidOCR
RapidocrOcrEngine._engine = RapidOCR()
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
except Exception:
RapidocrOcrEngine._unavailable = True
logger.warning(
"OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True
)
return ""
# 2. 渲染并识别
try:
import pypdfium2 as pdfium
scale = max(1.0, dpi / 72.0)
pdf = pdfium.PdfDocument(io.BytesIO(content))
total = min(len(pdf), max(1, max_pages))
page_texts: list[str] = []
for i in range(total):
page = pdf[i]
pil_image = page.render(scale=scale).to_pil()
result, _ = self._engine(pil_image)
if result:
lines = [
item[1]
for item in result
if item and len(item) >= 2 and item[1]
]
if lines:
page_texts.append("\n".join(lines))
pdf.close()
return "\n".join(page_texts).strip()
except Exception as exc:
logger.warning(
"PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True
)
return ""
# ---------------------------------------------------------------------------- #
# 备选 OCR 插件:tesseractlazy import,依赖缺失时不可用)
# ---------------------------------------------------------------------------- #
class TesseractOcrEngine:
"""pytesseract + pdfium OCR(备选)
需要系统安装 tesseract 二进制与语言包。
类级 _unavailable 单例,跨实例共享。
"""
_unavailable: bool = False
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
if self._unavailable:
return ""
try:
import pypdfium2 as pdfium
import pytesseract # type: ignore[import-untyped]
from PIL import Image # type: ignore[import-untyped]
except Exception:
TesseractOcrEngine._unavailable = True
logger.warning("tesseract 依赖不可用,降级返回空文本", exc_info=True)
return ""
try:
scale = max(1.0, dpi / 72.0)
pdf = pdfium.PdfDocument(io.BytesIO(content))
total = min(len(pdf), max(1, max_pages))
page_texts: list[str] = []
for i in range(total):
page = pdf[i]
pil_image = page.render(scale=scale).to_pil()
text = pytesseract.image_to_string(pil_image, lang="chi_sim+eng")
if text:
page_texts.append(text)
pdf.close()
return "\n".join(page_texts).strip()
except Exception as exc:
logger.warning(
"tesseract OCR 失败,降级返回空文本", error=str(exc), exc_info=True
)
return ""
class NoopOcrEngine:
"""关闭 OCR 的占位插件(strategy=none 时使用)"""
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
return ""
# ---------------------------------------------------------------------------- #
# 插件注册表
# ---------------------------------------------------------------------------- #
_PDF_PLUGINS: dict[str, type[PdfTextExtractor]] = {
"pypdf": PypdfTextExtractor,
"pdfplumber": PdfplumberTextExtractor,
}
_DOCX_PLUGINS: dict[str, type[DocxParser]] = {
"python_docx": PythonDocxParser,
}
_OCR_PLUGINS: dict[str, type[OcrEngine]] = {
"rapidocr": RapidocrOcrEngine,
"tesseract": TesseractOcrEngine,
"none": NoopOcrEngine,
}
def list_pdf_plugins() -> list[str]:
return list(_PDF_PLUGINS.keys())
def list_docx_plugins() -> list[str]:
return list(_DOCX_PLUGINS.keys())
def list_ocr_plugins() -> list[str]:
return list(_OCR_PLUGINS.keys())
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
# 进程级插件单例缓存
_pdf_plugin_cache: dict[str, PdfTextExtractor] = {}
_docx_plugin_cache: dict[str, DocxParser] = {}
_ocr_plugin_cache: dict[str, OcrEngine] = {}
def _get_pdf_plugin() -> PdfTextExtractor:
rt = get_runtime_settings()
name = rt.parsers.pdf.plugin or "pypdf"
if name in _pdf_plugin_cache:
return _pdf_plugin_cache[name]
cls = _PDF_PLUGINS.get(name)
if cls is None:
logger.warning("未知 PDF 插件,降级到 pypdf", requested=name)
cls = PypdfTextExtractor
name = "pypdf"
instance = cls()
_pdf_plugin_cache[name] = instance
return instance
def _get_docx_plugin() -> DocxParser:
rt = get_runtime_settings()
name = rt.parsers.docx.plugin or "python_docx"
if name in _docx_plugin_cache:
return _docx_plugin_cache[name]
cls = _DOCX_PLUGINS.get(name)
if cls is None:
logger.warning("未知 DOCX 插件,降级到 python_docx", requested=name)
cls = PythonDocxParser
name = "python_docx"
instance = cls()
_docx_plugin_cache[name] = instance
return instance
def _get_ocr_plugin() -> OcrEngine:
rt = get_runtime_settings()
name = rt.parsers.ocr.plugin or "rapidocr"
if name in _ocr_plugin_cache:
return _ocr_plugin_cache[name]
cls = _OCR_PLUGINS.get(name)
if cls is None:
logger.warning("未知 OCR 插件,降级到 rapidocr", requested=name)
cls = RapidocrOcrEngine
name = "rapidocr"
instance = cls()
_ocr_plugin_cache[name] = instance
return instance
def invalidate_parser_plugin_cache(kind: str | None = None) -> None:
"""清除插件缓存(runtime_settings 更新后调用)"""
if kind is None or kind == "pdf":
_pdf_plugin_cache.clear()
if kind is None or kind == "docx":
_docx_plugin_cache.clear()
if kind is None or kind == "ocr":
_ocr_plugin_cache.clear()
# 重置 RapidocrOcrEngine 类级标记,允许重新初始化
RapidocrOcrEngine._engine = None
RapidocrOcrEngine._unavailable = False
TesseractOcrEngine._unavailable = False
# ---------------------------------------------------------------------------- #
# 顶层解析函数
# ---------------------------------------------------------------------------- #
def _parse_pdf(content: bytes) -> str:
"""PDF 解析:优先文本层插件;文本层为空时降级 OCR 插件"""
rt = get_runtime_settings()
pdf_plugin = _get_pdf_plugin()
try:
text_layer = pdf_plugin.extract_text(content)
except Exception as exc:
logger.warning(
"PDF 文本层提取失败,尝试 OCR 降级",
plugin=rt.parsers.pdf.plugin,
error=str(exc),
exc_info=True,
)
text_layer = ""
# 文本层非空:直接返回
if text_layer:
return text_layer
# 文本层为空 → 尝试 OCR 降级
# 文本层为空 → OCR 降级
if not settings.pdf_ocr_enabled:
return ""
ocr_text = _ocr_pdf(
content, max_pages=settings.pdf_ocr_max_pages, dpi=settings.pdf_ocr_dpi
ocr_plugin = _get_ocr_plugin()
return ocr_plugin.ocr_pdf(
content,
max_pages=rt.parsers.ocr.params.get("max_pages", settings.pdf_ocr_max_pages),
dpi=rt.parsers.ocr.params.get("dpi", settings.pdf_ocr_dpi),
)
return ocr_text
def _ocr_pdf(content: bytes, max_pages: int, dpi: int) -> str:
"""对扫描件 PDF 跑 OCR:渲染每页 → 识别 → 拼接
返回空字符串的场景:依赖未安装 / 渲染或识别异常 / 无识别结果。
任何异常仅告警不抛出,由上游按"无法提取文本"处理。
"""
global _ocr_engine, _ocr_unavailable
if _ocr_unavailable:
return ""
# 1. 懒加载 OCR 引擎
if _ocr_engine is None:
try:
from rapidocr_onnxruntime import RapidOCR
_ocr_engine = RapidOCR()
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
except Exception:
_ocr_unavailable = True
logger.warning("OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True)
return ""
# 2. 渲染并识别
try:
import pypdfium2 as pdfium
scale = max(1.0, dpi / 72.0)
pdf = pdfium.PdfDocument(io.BytesIO(content))
total = min(len(pdf), max(1, max_pages))
page_texts: list[str] = []
for i in range(total):
page = pdf[i]
pil_image = page.render(scale=scale).to_pil()
result, _ = _ocr_engine(pil_image)
if result:
# result: [[box, text, score], ...],按行拼接
lines = [
item[1] for item in result if item and len(item) >= 2 and item[1]
]
if lines:
page_texts.append("\n".join(lines))
pdf.close()
return "\n".join(page_texts).strip()
except Exception as exc:
logger.warning("PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True)
return ""
def _parse_docx(content: bytes) -> str:
"""DOCX 段落文本拼接(不含表格/页眉页脚)"""
from docx import Document # type: ignore[import-untyped]
document = Document(io.BytesIO(content))
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
return "\n".join(parts).strip()
"""DOCX 解析"""
return _get_docx_plugin().extract_text(content)
# 扩展名 → 解析函数映射(启动时构建,避免每次请求重复构造)
# 扩展名 → 解析函数映射
_PARSERS: dict[str, Callable[[bytes], str]] = {
".txt": _parse_text,
".md": _parse_text,
+13 -37
View File
@@ -6,10 +6,12 @@
内存注册表为主(记录 status/created_at/updated_at/result/error),
Redis 为持久镜像(key: ingest_task:{task_id}),每次状态迁移同步写入;
Redis 不可用或写入失败仅记录 warning,不影响任务执行。
文本去重委托给 app.core.dedup.get_dedup_strategy(),按 runtime_settings.dedup
选择策略(none/sha256/simhash)。
"""
import asyncio
import hashlib
import uuid
from datetime import UTC, datetime
from enum import StrEnum
@@ -18,6 +20,7 @@ from typing import Any
import structlog
from app.config import Settings
from app.core.dedup import DEDUP_KEY_PREFIX, get_dedup_strategy
from app.core.ingestion import Ingester, IngestionError
from app.models.document import DocumentInput
from app.services.redis import RedisCache
@@ -26,8 +29,6 @@ logger = structlog.get_logger()
# Redis 任务状态 key 前缀
REDIS_KEY_PREFIX = "ingest_task:"
# Redis 文本去重 key 前缀(value 为已入库文档的 IngestionResult JSON
DEDUP_KEY_PREFIX = "dedup:sha256:"
class IngestTaskStatus(StrEnum):
@@ -71,17 +72,17 @@ class IngestTaskManager:
async def submit(self, doc: DocumentInput) -> str:
"""登记入库任务并后台执行,立即返回 task_id
文本去重:基于 doc.text 的 sha256 在 Redis 中查重;命中则直接复用旧
IngestionResult(仅置 deduplicated=True),不重跑流水线;未命中走原
异步入库流程,完成后写入去重记录供后续命中复用。Redis 不可用时跳过
去重,按原流程执行,不影响主流程
文本去重:按 runtime_settings.dedup 选择策略(none/sha256/simhash);
命中则直接复用旧 IngestionResult(仅置 deduplicated=True),不重跑流水线;
未命中走原异步入库流程,完成后写入去重记录供后续命中复用。
Redis 不可用或策略=none 时跳过,按原流程执行
"""
task_id = uuid.uuid4().hex
now = _utc_now_iso()
text_hash = hashlib.sha256(doc.text.encode("utf-8")).hexdigest()
dedup = get_dedup_strategy(self._redis)
# 1. 去重命中:直接置 done,复用旧结果,不调 _run
dedup_record = await self._lookup_dedup(text_hash)
dedup_record = await dedup.lookup(doc.text)
if dedup_record is not None:
result_dict = dict(dedup_record)
result_dict["deduplicated"] = True
@@ -111,7 +112,7 @@ class IngestTaskManager:
"error": None,
}
self._schedule_mirror(task_id)
background = asyncio.create_task(self._run(task_id, doc, text_hash))
background = asyncio.create_task(self._run(task_id, doc, dedup))
self._background_tasks.add(background)
background.add_done_callback(self._background_tasks.discard)
logger.info("入库任务已登记", task_id=task_id, title=doc.title)
@@ -151,7 +152,7 @@ class IngestTaskManager:
raise TimeoutError(f"入库任务 {task_id}{timeout}s 内未进入终态")
await asyncio.sleep(0.01)
async def _run(self, task_id: str, doc: DocumentInput, text_hash: str) -> None:
async def _run(self, task_id: str, doc: DocumentInput, dedup: Any) -> None:
"""后台执行入库:并发限流 + 阶段状态推进 + 结果/错误落账 + 去重记录写入"""
async with self._semaphore:
try:
@@ -185,34 +186,9 @@ class IngestTaskManager:
result=result_dict,
)
self._schedule_mirror(task_id)
await self._record_dedup(text_hash, result_dict)
await dedup.record(doc.text, result_dict)
logger.info("入库任务完成", task_id=task_id)
async def _lookup_dedup(self, text_hash: str) -> dict[str, Any] | None:
"""查询文本去重记录;Redis 不可用或异常时降级为未命中"""
if self._redis is None:
return None
try:
return await self._redis.get_json(f"{DEDUP_KEY_PREFIX}{text_hash}")
except Exception:
logger.warning(
"去重记录查询失败,降级为未命中", text_hash=text_hash, exc_info=True
)
return None
async def _record_dedup(self, text_hash: str, result_dict: dict[str, Any]) -> None:
"""写入文本去重记录(含完整 IngestionResult),供后续命中复用;失败仅告警"""
if self._redis is None:
return
try:
await self._redis.set_json(
f"{DEDUP_KEY_PREFIX}{text_hash}",
result_dict,
ttl=self._settings.ingest_task_ttl_done,
)
except Exception:
logger.warning("去重记录写入失败", text_hash=text_hash, exc_info=True)
def _finish_failed(self, task_id: str, error: dict[str, Any]) -> None:
"""将任务置为 failed 并记录错误信息"""
self._tasks[task_id].update(
+14 -8
View File
@@ -1,6 +1,7 @@
"""query 解析与分类路由模块
分层 RAG 在线侧第一步:用 Ollama 小模型将用户 query 解析为结构化 JSON
分层 RAG 在线侧第一步:用 LLMOllama 或 OpenAI 兼容服务,由
runtime_settings.models.query 决定)将用户 query 解析为结构化 JSON
(命中类目+置信度、rewrite 后 query、关键词),再由纯函数做路由决策:
- 高置信且命中类目数 <= 上限 → 按类目过滤检索
- 低置信 / 解析失败 / 命中类目过多 → 全库兜底(不丢召回)
@@ -16,8 +17,8 @@ import structlog
from pydantic import BaseModel, Field, ValidationError
from app.config import settings
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory
from app.services.ollama import OllamaClient
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory, load_taxonomy
from app.services.llm import LLMClient, create_llm_client
from app.services.redis import RedisCache, get_cache
logger = structlog.get_logger()
@@ -106,17 +107,22 @@ def decide_route(parsed: ParsedQuery, threshold: float, max_categories: int) ->
class QueryParser:
"""query 解析器:调用 Ollama 小模型将 query 解析为结构化 JSON"""
"""query 解析器:调用 LLM 将 query 解析为结构化 JSON"""
def __init__(
self, ollama: OllamaClient, taxonomy: list[TaxonomyCategory], cache: RedisCache | None = None
self,
ollama: LLMClient | None = None,
taxonomy: list[TaxonomyCategory] | None = None,
cache: RedisCache | None = None,
) -> None:
self.ollama = ollama
self.taxonomy = taxonomy
# 默认按 runtime_settings.models.query 选择 LLM 实现;
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("query")
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
# 解析结果缓存,缺省用全局单例;RedisCache 全操作容错,缓存不可用时退化为无缓存行为
self.cache = cache if cache is not None else get_cache()
# 可作为路由命中类目的名字集合(uncategorized 不可作为路由命中类目)
self._routable_names = {c.name for c in taxonomy if c.name != UNCATEGORIZED}
self._routable_names = {c.name for c in self.taxonomy if c.name != UNCATEGORIZED}
async def parse(self, query: str) -> ParsedQuery:
"""调用 LLM 将 query 解析为结构化结果
+7 -4
View File
@@ -1,6 +1,7 @@
"""检索结果 AI 总结
对检索返回的 chunk 命中结果,调用 Ollama 本地模型生成一段针对用户 query 的总结回答。
对检索返回的 chunk 命中结果,调用 LLMOllama 或 OpenAI 兼容服务,由
runtime_settings.models.query 决定)生成一段针对用户 query 的总结回答。
仅基于检索结果内容,不编造未提及的信息。
"""
@@ -8,7 +9,7 @@ import structlog
from app.config import settings
from app.models.search import SearchHit
from app.services.ollama import OllamaClient
from app.services.llm import LLMClient, create_llm_client
logger = structlog.get_logger()
@@ -16,8 +17,10 @@ logger = structlog.get_logger()
class ResultSummarizer:
"""检索结果总结器"""
def __init__(self, ollama: OllamaClient | None = None) -> None:
self.ollama = ollama or OllamaClient()
def __init__(self, ollama: LLMClient | None = None) -> None:
# 默认按 runtime_settings.models.query 选择 LLM 实现;
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("query")
async def summarize(self, query: str, hits: list[SearchHit]) -> str:
"""对检索结果生成针对 query 的总结
+2 -2
View File
@@ -21,7 +21,7 @@ from app.core.result_summarizer import ResultSummarizer
from app.core.sparse import SparseEncoder
from app.models.knowledge import load_taxonomy
from app.models.search import ExtractedInfo, SearchHit, SearchRequest, SearchResponse
from app.services.ollama import OllamaClient
from app.services.llm import create_llm_client
from app.services.qdrant import (
COLLECTION_CHUNKS,
COLLECTION_L1,
@@ -52,7 +52,7 @@ class Retriever:
) -> None:
self.qdrant = qdrant or QdrantService()
self.query_parser = query_parser or QueryParser(
ollama=OllamaClient(),
ollama=create_llm_client("query"),
taxonomy=load_taxonomy(settings.taxonomy_path),
)
self.embedding = embedding or create_embedding_service()
+224
View File
@@ -0,0 +1,224 @@
"""运行时可调配置(runtime settings
与 `app.config.settings`(启动时从环境变量加载、不可变)互补:本模块管理运行时
可通过管理后台动态调整的子集,持久化到 JSON 文件,启动时加载覆盖到内存单例。
设计要点:
- RuntimeSettings 只包含「可运行时调整」的字段(模型/解析插件/去重等),
不包含敏感或启动期固定的字段(端口、数据库连接等)。
- 持久化路径默认 `data/runtime_settings.json`,可由 env `RUNTIME_SETTINGS_PATH` 覆盖。
- 加载失败/文件缺失时回退到默认值,不阻塞启动。
- 写入采用「先临时文件后 rename」原子替换,避免半写损坏。
- 全模块只通过 `get_runtime_settings()` 访问单例,避免直接读 JSON。
"""
from __future__ import annotations
import json
import os
import tempfile
from pathlib import Path
from threading import RLock
from typing import Any, Literal
import structlog
from pydantic import BaseModel, Field
from app.config import settings
logger = structlog.get_logger()
# 默认持久化路径(相对工作目录);可由 env 覆盖
DEFAULT_RUNTIME_SETTINGS_PATH = "./data/runtime_settings.json"
_lock = RLock()
# ---------------------------------------------------------------------------- #
# 模型配置(文档总结 / 查询 / 分类 各一份独立配置)
# ---------------------------------------------------------------------------- #
class LlmProviderConfig(BaseModel):
"""单个用途的 LLM 提供方配置"""
provider: Literal["ollama", "openai_compatible"] = Field(
default="ollama", description="提供方:ollama 走 Ollama HTTPopenai_compatible 走 OpenAI 兼容 chat/completions"
)
base_url: str = Field(default="", description="服务地址;空则 ollama 用 settings.ollama_base_urlopenai_compatible 用 settings.openai_base_url")
api_key: str = Field(default="", description="API Key(仅 openai_compatible 需要;ollama 忽略)")
model: str = Field(default="", description="模型名;空则 ollama 用 settings.ollama_modelopenai_compatible 用 settings.embedding_model 同级(如 gpt-4o-mini")
timeout: float = Field(default=120.0, description="请求超时秒数")
temperature: float = Field(default=0.3, description="采样温度(0~2")
class ModelSettings(BaseModel):
"""模型相关运行时配置:三种用途独立配置"""
summarize: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档三级总结")
query: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="检索链路 query 解析与结果总结")
classify: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档分类判定")
# ---------------------------------------------------------------------------- #
# 解析插件配置
# ---------------------------------------------------------------------------- #
class PluginConfig(BaseModel):
"""单个解析插件的选择与参数"""
plugin: str = Field(default="", description="插件名;空则用默认")
params: dict[str, Any] = Field(default_factory=dict, description="插件参数(透传给插件实现)")
class ParserSettings(BaseModel):
"""解析插件运行时配置"""
ocr: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="rapidocr"), description="OCR 插件")
pdf: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="pypdf"), description="PDF 文本层提取插件")
docx: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="python_docx"), description="DOCX 解析插件")
# ---------------------------------------------------------------------------- #
# 去重策略配置
# ---------------------------------------------------------------------------- #
class DedupSettings(BaseModel):
"""文本去重策略运行时配置"""
strategy: Literal["none", "sha256", "simhash"] = Field(
default="sha256", description="去重策略:none 关闭;sha256 精确匹配;simhash 近似匹配"
)
simhash_threshold: int = Field(default=3, description="simhash 海明距离阈值(仅 strategy=simhash 生效,0~64")
ttl_seconds: int = Field(default=86400, description="去重记录 Redis 保留秒数")
# ---------------------------------------------------------------------------- #
# 顶层 RuntimeSettings
# ---------------------------------------------------------------------------- #
class RuntimeSettings(BaseModel):
"""运行时可调配置顶层模型"""
models: ModelSettings = Field(default_factory=ModelSettings)
parsers: ParserSettings = Field(default_factory=ParserSettings)
dedup: DedupSettings = Field(default_factory=DedupSettings)
# ---------------------------------------------------------------------------- #
# 单例 + 持久化
# ---------------------------------------------------------------------------- #
_runtime_settings: RuntimeSettings | None = None
def _resolve_path() -> Path:
"""解析持久化文件路径:env RUNTIME_SETTINGS_PATH > settings 自定义 > 默认"""
path_str = os.environ.get("RUNTIME_SETTINGS_PATH", "") or getattr(settings, "runtime_settings_path", "") or DEFAULT_RUNTIME_SETTINGS_PATH
return Path(path_str).expanduser().resolve()
def _default_with_env_fallback() -> RuntimeSettings:
"""构造默认 RuntimeSettings,并把启动 env 中已有的模型相关字段填充进去
这样首次启动(无持久化文件)时,UI 显示的不是空字符串而是 env 当前值。
"""
cfg = RuntimeSettings()
# 模型默认值沿用 env
cfg.models.summarize.base_url = settings.ollama_base_url
cfg.models.summarize.model = settings.ollama_model
cfg.models.query.base_url = settings.ollama_base_url
cfg.models.query.model = settings.ollama_model
cfg.models.classify.base_url = settings.ollama_base_url
cfg.models.classify.model = settings.ollama_model
# 若 env 提供了 OpenAI key/url,预填到 openai_compatible 字段方便切换
if settings.openai_api_key:
for usage in ("summarize", "query", "classify"):
getattr(cfg.models, usage).api_key = settings.openai_api_key
getattr(cfg.models, usage).base_url = settings.openai_base_url if settings.openai_base_url else getattr(cfg.models, usage).base_url
return cfg
def load_runtime_settings() -> RuntimeSettings:
"""从磁盘加载 RuntimeSettings;文件缺失或损坏时回退到默认值(带 env 兜底)"""
path = _resolve_path()
try:
if path.exists():
data = json.loads(path.read_text(encoding="utf-8"))
return RuntimeSettings.model_validate(data)
except Exception:
logger.warning("RuntimeSettings 加载失败,回退默认值", path=str(path), exc_info=True)
return _default_with_env_fallback()
def save_runtime_settings(cfg: RuntimeSettings) -> None:
"""原子写入 RuntimeSettings 到磁盘(先临时文件后 rename)"""
path = _resolve_path()
path.parent.mkdir(parents=True, exist_ok=True)
data = cfg.model_dump(mode="json")
# 写到同目录临时文件再 rename,避免半写损坏
fd, tmp_path = tempfile.mkstemp(prefix=".runtime_settings.", suffix=".tmp", dir=str(path.parent))
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
os.replace(tmp_path, path)
except Exception:
try:
os.unlink(tmp_path)
except OSError:
pass
raise
def get_runtime_settings() -> RuntimeSettings:
"""获取 RuntimeSettings 单例(首次调用时加载)"""
global _runtime_settings
with _lock:
if _runtime_settings is None:
_runtime_settings = load_runtime_settings()
return _runtime_settings
def update_runtime_settings(patch: dict[str, Any]) -> RuntimeSettings:
"""以 patch 字典更新 RuntimeSettings 并持久化
支持部分更新(顶层键可缺失,缺省保留原值)。例如:
update_runtime_settings({"models": {"summarize": {"model": "qwen2.5:3b"}}})
"""
with _lock:
current = get_runtime_settings()
merged = current.model_dump(mode="json")
_deep_merge(merged, patch)
new_cfg = RuntimeSettings.model_validate(merged)
save_runtime_settings(new_cfg)
# 替换单例后立即返回,新值对所有后续读取生效
global _runtime_settings
_runtime_settings = new_cfg
logger.info("RuntimeSettings 已更新并持久化", path=str(_resolve_path()))
return new_cfg
def _deep_merge(target: dict[str, Any], patch: dict[str, Any]) -> None:
"""递归把 patch 合并到 target(同 key 字典则递归,否则覆盖)"""
for k, v in patch.items():
if k in target and isinstance(target[k], dict) and isinstance(v, dict):
_deep_merge(target[k], v)
else:
target[k] = v
def reload_runtime_settings() -> RuntimeSettings:
"""强制从磁盘重新加载(管理后台触发)"""
with _lock:
global _runtime_settings
_runtime_settings = load_runtime_settings()
return _runtime_settings
def reset_runtime_settings() -> RuntimeSettings:
"""重置为默认值并持久化(管理后台触发)"""
with _lock:
new_cfg = _default_with_env_fallback()
save_runtime_settings(new_cfg)
global _runtime_settings
_runtime_settings = new_cfg
return new_cfg
+7 -4
View File
@@ -1,6 +1,7 @@
"""文档三级总结模块
通过 Ollama 本地小模型对文档进行分级总结:
通过 LLMOllama 或 OpenAI 兼容服务,由 runtime_settings.models.summarize 决定)
对文档进行分级总结:
- L1: 总结(一句话高度概括)
- L2: 大纲(主要章节和关键主题)
- L3: 内容大纲(每个章节的详细内容摘要)
@@ -12,7 +13,7 @@ import structlog
from app.core.headings import parse_headings, render_outline
from app.models.document import DocumentSummary, SummaryLevel
from app.services.ollama import OllamaClient
from app.services.llm import LLMClient, create_llm_client
logger = structlog.get_logger()
@@ -23,8 +24,10 @@ MIN_TEXT_LENGTH_FOR_L3 = 500
class Summarizer:
"""文档三级总结器"""
def __init__(self, ollama: OllamaClient | None = None) -> None:
self.ollama = ollama or OllamaClient()
def __init__(self, ollama: LLMClient | None = None) -> None:
# 默认按 runtime_settings 选择 LLM 实现(Ollama 或 OpenAI 兼容);
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("summarize")
async def summarize(self, text: str, *, title: str = "") -> DocumentSummary:
"""对文档文本进行三级总结
+50 -4
View File
@@ -5,13 +5,14 @@ from pathlib import Path
import structlog
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import FileResponse, JSONResponse
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
from app.api.response import ApiError, error
from app.api.v1.auth import router as auth_router
from app.api.v1.document import router as document_router
from app.api.v1.knowledge import router as knowledge_router
from app.api.v1.search import router as search_router
from app.api.v1.settings import router as settings_router
from app.config import settings
from app.core.auth import ensure_default_admin
from app.services.qdrant import QdrantService
@@ -53,6 +54,7 @@ app.include_router(auth_router)
app.include_router(search_router)
app.include_router(document_router)
app.include_router(knowledge_router)
app.include_router(settings_router)
@app.exception_handler(ApiError)
@@ -84,10 +86,54 @@ async def health() -> dict[str, str]:
return {"status": "ok"}
_ADMIN_HTML = Path(__file__).resolve().parent / "static" / "admin.html"
_STATIC_DIR = Path(__file__).resolve().parent / "static"
_ADMIN_HTML = _STATIC_DIR / "admin.html"
_ADMIN_SPA_DIR = _STATIC_DIR / "admin"
def _spa_index() -> Path | None:
"""SPA 入口文件路径;不存在返回 None(回退旧 admin.html"""
index = _ADMIN_SPA_DIR / "index.html"
return index if index.is_file() else None
@app.get("/admin", include_in_schema=False)
async def admin_page() -> FileResponse:
"""管理后台单页(单文件静态 HTML,零外部依赖)"""
async def admin_page():
"""管理后台:优先服务 Vue SPA,回退到旧单文件 admin.html"""
spa = _spa_index()
if spa is not None:
return FileResponse(spa, media_type="text/html")
return FileResponse(_ADMIN_HTML, media_type="text/html")
@app.get("/admin/", include_in_schema=False)
async def admin_page_trailing_slash():
"""带斜杠的 /admin/ 重定向到 /admin"""
return RedirectResponse(url="/admin", status_code=307)
@app.get("/admin/{rest:path}", include_in_schema=False)
async def admin_spa(rest: str):
"""SPA 静态资源与客户端路由兜底
- 真实文件(assets/*.js, *.css 等)→ 直接返回
- 其余路径(/overview, /documents 等客户端路由)→ 返回 index.html
- SPA 目录不存在时 → 404(旧 admin.html 无子路由需求)
"""
spa_dir = _ADMIN_SPA_DIR
if not spa_dir.is_dir():
return JSONResponse(
status_code=404, content={"code": 1002, "message": "Not found"}
)
# 安全校验:防止路径越界
file_path = (spa_dir / rest).resolve()
try:
file_path.relative_to(spa_dir.resolve())
except ValueError:
return FileResponse(spa_dir / "index.html", media_type="text/html")
if file_path.is_file():
return FileResponse(file_path)
# SPA 客户端路由兜底
return FileResponse(spa_dir / "index.html", media_type="text/html")
+179
View File
@@ -0,0 +1,179 @@
"""LLM 客户端抽象层
提供统一的 `generate(prompt, json_mode)` 接口,底层实现可切换:
- OllamaLLMClient:走 Ollama HTTP /api/generate(本地或远程 Ollama 服务)
- OpenAICompatibleLLMClient:走 OpenAI 兼容 /v1/chat/completionsOpenAI / DeepSeek / 智谱 / Qwen API 等)
工厂 `create_llm_client(purpose)` 根据 runtime_settings 选择实现与参数。
所有客户端共享同一接口,业务侧无需感知底层协议差异。
"""
from __future__ import annotations
from typing import Literal, Protocol, runtime_checkable
import httpx
import structlog
from app.config import settings
from app.core.runtime_settings import LlmProviderConfig, get_runtime_settings
from app.services.ollama import OllamaClient
logger = structlog.get_logger()
LlmPurpose = Literal["summarize", "query", "classify"]
@runtime_checkable
class LLMClient(Protocol):
"""LLM 客户端统一接口"""
async def generate(self, prompt: str, json_mode: bool = False) -> str:
"""生成文本
Args:
prompt: 输入提示词
json_mode: True 时约束输出为 JSON(不支持时降级为普通生成)
Returns:
生成的文本内容
"""
...
async def is_available(self) -> bool:
"""检查服务是否可用"""
...
class OllamaLLMClient:
"""Ollama HTTP API 客户端(包装现有 OllamaClient,便于统一接口)"""
def __init__(self, base_url: str, model: str, timeout: float = 120.0) -> None:
self._inner = OllamaClient(base_url=base_url, model=model, timeout=timeout)
async def generate(self, prompt: str, json_mode: bool = False) -> str:
return await self._inner.generate(prompt, json_mode=json_mode)
async def is_available(self) -> bool:
return await self._inner.is_available()
class OpenAICompatibleLLMClient:
"""OpenAI 兼容 chat/completions 客户端
适用于 OpenAI 官方 API、DeepSeek、智谱 ChatGLM、Qwen DashScope、Moonshot 等
所有兼容 OpenAI /v1/chat/completions 协议的服务。
"""
def __init__(self, base_url: str, api_key: str, model: str, timeout: float = 120.0, temperature: float = 0.3) -> None:
self.base_url = base_url.rstrip("/")
self.api_key = api_key
self.model = model
self.timeout = timeout
self.temperature = temperature
async def generate(self, prompt: str, json_mode: bool = False) -> str:
url = f"{self.base_url}/chat/completions"
payload: dict = {
"model": self.model,
"messages": [{"role": "user", "content": prompt}],
"temperature": self.temperature,
"stream": False,
}
if json_mode:
payload["response_format"] = {"type": "json_object"}
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
async with httpx.AsyncClient(timeout=self.timeout) as client:
resp = await client.post(url, json=payload, headers=headers)
resp.raise_for_status()
data = resp.json()
# OpenAI 标准响应结构:choices[0].message.content
choices = data.get("choices") or []
if not choices:
logger.warning("OpenAI 兼容响应无 choices", model=self.model, raw_keys=list(data.keys()))
return ""
message = choices[0].get("message") or {}
content = message.get("content") or ""
logger.debug("OpenAI 兼容生成完成", model=self.model, output_length=len(content))
return content
async def is_available(self) -> bool:
"""简单探测:调 /models 列表接口(多数 OpenAI 兼容服务支持)"""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(
f"{self.base_url}/models",
headers={"Authorization": f"Bearer {self.api_key}"},
)
return resp.status_code == 200
except httpx.HTTPError:
return False
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
# 进程级客户端缓存(避免每请求新建 httpx client
_client_cache: dict[str, LLMClient] = {}
def _resolve_provider_config(purpose: LlmPurpose) -> LlmProviderConfig:
"""从 runtime_settings 取指定用途的 LLM 配置"""
rt = get_runtime_settings()
return getattr(rt.models, purpose)
def _build_client(purpose: LlmPurpose) -> LLMClient:
"""按 runtime_settings 构造 LLM 客户端"""
cfg = _resolve_provider_config(purpose)
if cfg.provider == "ollama":
base_url = cfg.base_url or settings.ollama_base_url
model = cfg.model or settings.ollama_model
return OllamaLLMClient(base_url=base_url, model=model, timeout=cfg.timeout)
if cfg.provider == "openai_compatible":
base_url = cfg.base_url or settings.openai_base_url
api_key = cfg.api_key or settings.openai_api_key
# openai_compatible 默认模型:若 cfg.model 为空,退化到 OpenAI 通用 chat 模型
model = cfg.model or "gpt-4o-mini"
if not api_key:
logger.warning("OpenAI 兼容 provider 缺少 api_key,调用大概率会失败", purpose=purpose)
return OpenAICompatibleLLMClient(
base_url=base_url, api_key=api_key, model=model, timeout=cfg.timeout, temperature=cfg.temperature
)
raise ValueError(f"未知 LLM provider: {cfg.provider}")
def create_llm_client(purpose: LlmPurpose, *, use_cache: bool = True) -> LLMClient:
"""创建指定用途的 LLM 客户端
Args:
purpose: 用途(summarize / query / classify
use_cache: True 时复用进程级客户端单例(默认);False 每次新建(测试用)
Returns:
LLMClient 实例
"""
if not use_cache:
return _build_client(purpose)
cache_key = f"{purpose}"
if cache_key not in _client_cache:
_client_cache[cache_key] = _build_client(purpose)
return _client_cache[cache_key]
def invalidate_llm_client_cache(purpose: LlmPurpose | None = None) -> None:
"""清除客户端缓存(runtime_settings 更新后调用,确保后续读取新配置)"""
if purpose is None:
_client_cache.clear()
else:
_client_cache.pop(purpose, None)