feat: 完成全量功能开发,包括前端管理后台与后端服务优化

此提交实现了完整的知识库管理系统:
1. 新增Vue3 + Antd Vue前端管理后台,包含登录、文档管理、检索、类目设置等完整页面
2. 重构后端LLM调用抽象层,支持Ollama与OpenAI兼容服务动态切换
3. 调整默认嵌入模型配置为本地bge-m3模式
4. 优化入库任务去重逻辑与缓存清理机制
5. 完善Docker镜像构建与docker-compose部署配置
6. 修复多项测试用例与兼容性问题
7. 新增运行时配置API,支持动态调整系统参数
This commit is contained in:
2026-07-31 12:05:25 +08:00
parent fdb664e546
commit 2ab8b56a01
52 changed files with 7030 additions and 171 deletions
+16
View File
@@ -0,0 +1,16 @@
.git
.venv
__pycache__
*.pyc
.pytest_cache
htmlcov
.coverage
data
logs
*.log
.env
.env.local
docker-compose.override.yml
frontend/node_modules
frontend/dist
app/static/admin
+3
View File
@@ -6,6 +6,9 @@ dist/
build/
.venv/
# Frontend
node_modules/
# Env
.env
.env.local
+17
View File
@@ -1,3 +1,17 @@
# ---------- Stage 1: 前端构建 ----------
FROM node:20-slim AS frontend-builder
WORKDIR /frontend
# 依赖层(利用 Docker 层缓存)
COPY frontend/package.json frontend/package-lock.json ./
RUN npm ci
# 源码 + 构建
COPY frontend/ ./
RUN npm run build
# ---------- Stage 2: Python 后端 ----------
FROM python:3.12-slim AS base
WORKDIR /app
@@ -13,6 +27,9 @@ RUN uv sync --frozen --no-dev
COPY app/ app/
COPY scripts/ scripts/
# 前端构建产物 → 后端静态目录(SPA 由 /admin 路由服务)
COPY --from=frontend-builder /frontend/dist /app/app/static/admin
EXPOSE 8000
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
+114
View File
@@ -0,0 +1,114 @@
"""运行时配置 APIGET/PUT /api/v1/settings、GET /api/v1/settings/schema、POST /api/v1/settings/reset
GET 任何登录用户可读;PUT/reset 需 admin。PUT 后会清空 LLM 客户端 / 解析插件 /
去重策略三处进程级缓存,使新配置立即对后续请求生效。
"""
from typing import Any
import structlog
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from app.api.response import ApiError, ok
from app.core.auth import AuthUser, get_current_user, require_admin
from app.core.dedup import invalidate_dedup_strategy_cache
from app.core.file_parser import (
list_docx_plugins,
list_ocr_plugins,
list_pdf_plugins,
invalidate_parser_plugin_cache,
)
from app.core.runtime_settings import (
RuntimeSettings,
get_runtime_settings,
reset_runtime_settings,
update_runtime_settings,
)
from app.services.llm import invalidate_llm_client_cache
logger = structlog.get_logger()
router = APIRouter(prefix="/api/v1", tags=["settings"])
class SettingsUpdateRequest(BaseModel):
"""Settings PATCH body:任意子树可缺省,缺省字段保留原值
例:{"models": {"summarize": {"model": "qwen2.5:3b"}}}、
{"parsers": {"ocr": {"plugin": "tesseract"}}}、
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
"""
models: dict[str, Any] | None = None
parsers: dict[str, Any] | None = None
dedup: dict[str, Any] | None = None
@router.get("/settings")
async def get_settings(user: AuthUser = Depends(get_current_user)) -> dict[str, Any]:
"""返回当前 RuntimeSettings(任何登录用户可读)"""
cfg = get_runtime_settings()
return ok(cfg.model_dump(mode="json"))
@router.put("/settings")
async def update_settings(
body: SettingsUpdateRequest,
user: AuthUser = Depends(require_admin),
) -> dict[str, Any]:
"""部分更新 RuntimeSettings(仅 admin
更新成功后清空 LLM 客户端 / 解析插件 / 去重策略三处进程级缓存,
使新配置立即对后续请求生效。
"""
patch = body.model_dump(exclude_none=True)
if not patch:
raise ApiError(1001, "请求体为空,未提供任何待更新字段")
try:
cfg = update_runtime_settings(patch)
except Exception as exc:
logger.error("RuntimeSettings 更新失败", error=str(exc), exc_info=True)
raise ApiError(2000, f"配置更新失败: {exc}") from exc
# 清缓存:让后续读取拿到新配置
invalidate_llm_client_cache()
invalidate_parser_plugin_cache()
invalidate_dedup_strategy_cache()
logger.info("RuntimeSettings 已更新并清理缓存", operator=user.username)
return ok(cfg.model_dump(mode="json"))
@router.get("/settings/schema")
async def get_settings_schema(
user: AuthUser = Depends(get_current_user),
) -> dict[str, Any]:
"""返回可选插件与策略列表(前端 Settings 页渲染选项用)"""
return ok(
{
"llm_providers": ["ollama", "openai_compatible"],
"pdf_plugins": list_pdf_plugins(),
"docx_plugins": list_docx_plugins(),
"ocr_plugins": list_ocr_plugins(),
"dedup_strategies": ["none", "sha256", "simhash"],
}
)
@router.post("/settings/reset")
async def reset_settings(
user: AuthUser = Depends(require_admin),
) -> dict[str, Any]:
"""重置 RuntimeSettings 为默认值(仅 admin),同时清缓存"""
try:
cfg = reset_runtime_settings()
except Exception as exc:
logger.error("RuntimeSettings 重置失败", error=str(exc), exc_info=True)
raise ApiError(2000, f"配置重置失败: {exc}") from exc
invalidate_llm_client_cache()
invalidate_parser_plugin_cache()
invalidate_dedup_strategy_cache()
logger.info("RuntimeSettings 已重置为默认值", operator=user.username)
return ok(cfg.model_dump(mode="json"))
+3 -2
View File
@@ -9,11 +9,12 @@ class Settings(BaseSettings):
log_level: str = "info"
# 嵌入模型
embedding_provider: str = "openai" # openai | local
embedding_provider: str = "local" # openai | local
openai_api_key: str = ""
openai_base_url: str = "https://api.openai.com/v1"
embedding_model: str = "text-embedding-3-small"
embedding_dimension: int = 1536
# bge-m3(本地 Ollama 嵌入)维度为 1024;切换 openai provider 时需同步改为 1536
embedding_dimension: int = 1024
# Ollama 本地模型(用于文档三级总结)
ollama_base_url: str = "http://localhost:11434"
+7 -4
View File
@@ -1,6 +1,7 @@
"""文档分类器
入库链路第二步:基于 L1 总结,用 Ollama 小模型将文档判定为 taxonomy 中的
入库链路第二步:基于 L1 总结,用 LLMOllama 或 OpenAI 兼容服务,由
runtime_settings.models.classify 决定)将文档判定为 taxonomy 中的
主类目 + 附加标签:
- LLM 输出解析失败 / 类目名不在 taxonomy → 归 uncategorizedconfidence=0.0
- 置信度低于阈值 → 主类目归 uncategorized,候选类目名保留进 tags(软召回用)
@@ -13,7 +14,7 @@ import structlog
from app.config import settings
from app.models.knowledge import UNCATEGORIZED, CategoryResult, TaxonomyCategory, load_taxonomy
from app.services.ollama import OllamaClient
from app.services.llm import LLMClient, create_llm_client
logger = structlog.get_logger()
@@ -47,8 +48,10 @@ def _extract_json(raw: str) -> dict | None:
class Classifier:
"""文档分类器:将 L1 总结判定为 taxonomy 主类目 + 附加标签"""
def __init__(self, ollama: OllamaClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
self.ollama = ollama or OllamaClient()
def __init__(self, ollama: LLMClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
# 默认按 runtime_settings.models.classify 选择 LLM 实现;
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("classify")
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
# 合法类目名集合(含 uncategorized
self._valid_names = {c.name for c in self.taxonomy}
+216
View File
@@ -0,0 +1,216 @@
"""文本去重策略工厂
支持三种策略:
- none:关闭去重,每次都跑完整流水线
- sha256:精确匹配(hash 完全相同才算重复)
- simhash:近似匹配(海明距离 <= 阈值视为重复)
每种策略实现 DedupStrategy 协议:lookup(text) -> dict|None、record(text, result) -> None。
工厂 get_dedup_strategy() 根据 runtime_settings.dedup 选择实现。
所有策略共享同一 Redis key 前缀,但 key 后缀按策略区分避免冲突。
"""
from __future__ import annotations
import hashlib
from typing import Any, Protocol, runtime_checkable
import structlog
from app.core.runtime_settings import get_runtime_settings
from app.services.redis import RedisCache
logger = structlog.get_logger()
# Redis 去重 key 前缀,按策略分桶
_DEDUP_KEY_PREFIX = "dedup:"
@runtime_checkable
class DedupStrategy(Protocol):
"""去重策略协议"""
async def lookup(self, text: str) -> dict[str, Any] | None:
"""查询文本是否已入库;命中返回旧 IngestionResultdict),未命中返回 None"""
...
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
"""记录文本已入库,供后续命中复用"""
...
class NoopDedupStrategy:
"""关闭去重:永远不命中,也不记录"""
async def lookup(self, text: str) -> dict[str, Any] | None:
return None
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
return None
class Sha256DedupStrategy:
"""SHA256 精确去重:完全相同文本才算重复"""
def __init__(self, redis: RedisCache, ttl_seconds: int) -> None:
self._redis = redis
self._ttl = ttl_seconds
def _key(self, text: str) -> str:
h = hashlib.sha256(text.encode("utf-8")).hexdigest()
return f"{_DEDUP_KEY_PREFIX}sha256:{h}"
async def lookup(self, text: str) -> dict[str, Any] | None:
try:
return await self._redis.get_json(self._key(text))
except Exception:
logger.warning("sha256 去重查询失败,降级未命中", exc_info=True)
return None
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
try:
await self._redis.set_json(self._key(text), result_dict, ttl=self._ttl)
except Exception:
logger.warning("sha256 去重记录写入失败", exc_info=True)
# SimHash 实现(64 位)
_MASK_64 = (1 << 64) - 1
def _simhash(text: str, token_size: int = 4) -> int:
"""计算文本 64 位 simhash
简化实现:按 token_size 字符滑窗分词,每段 md5 → 128 位 → 取低 64 位作为 hash,
逐位加权(hash 该位为 1 则 +1,为 0 则 -1),最终符号位定 1/0。
"""
if not text:
return 0
tokens = [text[i : i + token_size] for i in range(0, len(text), token_size)]
weights = [1] * 64
vec = [0] * 64
for token in tokens:
h = int(hashlib.md5(token.encode("utf-8")).hexdigest(), 16) & _MASK_64
for i in range(64):
bit = (h >> i) & 1
vec[i] += weights[i] if bit else -weights[i]
fingerprint = 0
for i in range(64):
if vec[i] > 0:
fingerprint |= (1 << i)
return fingerprint
def _hamming_distance(a: int, b: int) -> int:
return bin((a ^ b) & _MASK_64).count("1")
class SimhashDedupStrategy:
"""SimHash 近似去重:海明距离 <= 阈值视为重复
Redis 中存所有已入库文本的 simhashvalue 为 IngestionResult + simhash)。
lookup 时遍历所有候选 simhash 算海明距离,找最近的一个 <= 阈值则命中。
注:本实现为简化版,全表扫描,适合中小规模知识库;超大规模需换 LSH 索引。
"""
INDEX_KEY = f"{_DEDUP_KEY_PREFIX}simhash:index" # list 形式存所有 simhash+key
def __init__(self, redis: RedisCache, ttl_seconds: int, threshold: int = 3) -> None:
self._redis = redis
self._ttl = ttl_seconds
self._threshold = max(0, min(64, threshold))
async def lookup(self, text: str) -> dict[str, Any] | None:
try:
fingerprint = _simhash(text)
# 简化:扫描所有 simhash 记录找最近的
# 用 set:dedup:simhash:fps 存所有 fingerprint,每个 fp 对应一个 record key
# 这里用 list key 简化(适合小规模)
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
entries = index.get("entries", [])
best_dist = self._threshold + 1
best_key: str | None = None
for entry in entries:
fp = entry.get("fingerprint")
key = entry.get("key")
if fp is None or key is None:
continue
dist = _hamming_distance(fingerprint, int(fp))
if dist <= self._threshold and dist < best_dist:
best_dist = dist
best_key = key
if best_key is None:
return None
return await self._redis.get_json(best_key)
except Exception:
logger.warning("simhash 去重查询失败,降级未命中", exc_info=True)
return None
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
try:
fingerprint = _simhash(text)
key = f"{_DEDUP_KEY_PREFIX}simhash:{fingerprint:016x}"
# 1. 写记录
await self._redis.set_json(key, result_dict, ttl=self._ttl)
# 2. 更新索引
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
entries = index.get("entries", [])
entries.append({"fingerprint": fingerprint, "key": key})
await self._redis.set_json(self.INDEX_KEY, {"entries": entries}, ttl=self._ttl)
except Exception:
logger.warning("simhash 去重记录写入失败", exc_info=True)
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
_strategy_cache: DedupStrategy | None = None
_strategy_signature: tuple[str, int, int] | None = None # (strategy, ttl, threshold)
def get_dedup_strategy(redis: RedisCache | None) -> DedupStrategy:
"""按 runtime_settings.dedup 构造去重策略
Redis 不可用时降级为 NoopDedupStrategy(关闭去重),不影响主流程。
策略配置变更时自动重建单例。
"""
global _strategy_cache, _strategy_signature
rt = get_runtime_settings()
sig = (rt.dedup.strategy, rt.dedup.ttl_seconds, rt.dedup.simhash_threshold)
if _strategy_cache is not None and _strategy_signature == sig:
return _strategy_cache
if redis is None or rt.dedup.strategy == "none":
strategy: DedupStrategy = NoopDedupStrategy()
elif rt.dedup.strategy == "sha256":
strategy = Sha256DedupStrategy(redis=redis, ttl_seconds=rt.dedup.ttl_seconds)
elif rt.dedup.strategy == "simhash":
strategy = SimhashDedupStrategy(
redis=redis, ttl_seconds=rt.dedup.ttl_seconds, threshold=rt.dedup.simhash_threshold
)
else:
logger.warning("未知去重策略,降级为 noop", requested=rt.dedup.strategy)
strategy = NoopDedupStrategy()
_strategy_cache = strategy
_strategy_signature = sig
return strategy
def invalidate_dedup_strategy_cache() -> None:
"""清除策略缓存(runtime_settings 更新后调用)"""
global _strategy_cache, _strategy_signature
_strategy_cache = None
_strategy_signature = None
def compute_text_hash(text: str) -> str:
"""计算文本哈希(兼容旧接口,sha256 策略下与 dedup key 一致)"""
return hashlib.sha256(text.encode("utf-8")).hexdigest()
# 兼容旧测试与外部引用:保留 DEDUP_KEY_PREFIX 导出
DEDUP_KEY_PREFIX = _DEDUP_KEY_PREFIX
+346 -87
View File
@@ -1,14 +1,19 @@
"""多格式文件文本提取:按扩展名分发到对应解析器
"""多格式文件文本提取:按扩展名分发到对应解析器(插件化)
支持的扩展名:
- .txt / .mdUTF-8 解码(errors="replace" 兜底)
- .html / .htm:标准库 html.parser 剥离标签提取可见文本
- .pdfpypdf 逐页 extract_text 拼接;文本层为空(扫描件/图片型)时
自动降级为 OCRpypdfium2 渲染 + rapidocr-onnxruntime 识别
- .docxpython-docx 段落文本拼接(不含表格/页眉页脚)
- .pdfPDF 文本层提取插件(默认 pypdf);文本层为空(扫描件)时降级到 OCR 插件
- .docxDOCX 解析插件(默认 python-docx
插件化设计:
- PdfTextExtractor / DocxParser / OcrEngine 三个 Protocol
- 每种插件类型有注册表 + 默认实现 + 工厂
- 工厂根据 runtime_settings.parsers 选择具体插件
- 未注册或依赖缺失时降级到默认插件并告警
未识别扩展名抛 ValueError("不支持的文件类型: {ext}")
解析异常统一包装为 ValueError("文件解析失败: {detail}"),原异常链式保留
解析异常统一包装为 ValueError("文件解析失败: {detail}")。
"""
from __future__ import annotations
@@ -17,17 +22,19 @@ import io
from collections.abc import Callable
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from typing import Any, Protocol, runtime_checkable
import structlog
from app.config import settings
from app.core.runtime_settings import get_runtime_settings
logger = structlog.get_logger()
# 模块级懒加载 OCR 引擎单例(首次调用时初始化,避免无扫描件场景白白下载模型)
_ocr_engine: Any | None = None
_ocr_unavailable: bool = False # 标记 OCR 依赖不可用,后续直接跳过避免重复尝试
# ---------------------------------------------------------------------------- #
# HTML / Text 解析(无插件化需求,保留原实现)
# ---------------------------------------------------------------------------- #
class _VisibleTextExtractor(HTMLParser):
@@ -51,13 +58,11 @@ class _VisibleTextExtractor(HTMLParser):
self._parts.append(data)
def get_text(self) -> str:
# 块级标签间用空格连接,再折叠多余空白
text = " ".join(self._parts)
return " ".join(text.split())
def _decode_html(content: bytes) -> str:
"""HTML 内容解码:优先 utf-8(带 BOM),失败回退 latin-1"""
try:
return content.decode("utf-8-sig")
except UnicodeDecodeError:
@@ -65,108 +70,362 @@ def _decode_html(content: bytes) -> str:
def _parse_text(content: bytes) -> str:
"""UTF-8 解码(errors=replace 兜底),保留原字符"""
return content.decode("utf-8", errors="replace")
def _parse_html(content: bytes) -> str:
"""HTML 剥离标签,保留可见文本"""
parser = _VisibleTextExtractor()
parser.feed(_decode_html(content))
parser.close()
return parser.get_text()
def _parse_pdf(content: bytes) -> str:
"""PDF 解析:优先 pypdf extract_text;文本层为空(扫描件)时降级 OCR
# ---------------------------------------------------------------------------- #
# 插件协议
# ---------------------------------------------------------------------------- #
OCR 流程:pypdfium2 渲染每页为 PIL Image → rapidocr-onnxruntime 识别 →
拼接每页识别出的文本。受 settings.pdf_ocr_* 控制:开关、最大页数、DPI。
OCR 依赖未安装或运行异常时降级返回空字符串(由上游 upload 端点拒绝入库)。
@runtime_checkable
class PdfTextExtractor(Protocol):
"""PDF 文本层提取插件协议"""
def extract_text(self, content: bytes) -> str:
"""从 PDF 二进制内容提取文本层;无文本层返回空字符串"""
...
@runtime_checkable
class DocxParser(Protocol):
"""DOCX 解析插件协议"""
def extract_text(self, content: bytes) -> str:
"""从 DOCX 二进制内容提取段落文本"""
...
@runtime_checkable
class OcrEngine(Protocol):
"""OCR 引擎插件协议"""
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
"""对 PDF 跑 OCR,返回识别文本;失败返回空字符串"""
...
# ---------------------------------------------------------------------------- #
# 默认 PDF 插件:pypdf
# ---------------------------------------------------------------------------- #
class PypdfTextExtractor:
"""pypdf 文本层提取(默认)"""
def extract_text(self, content: bytes) -> str:
from pypdf import PdfReader
reader = PdfReader(io.BytesIO(content))
parts: list[str] = []
for page in reader.pages:
text = page.extract_text() or ""
if text:
parts.append(text)
return "\n".join(parts).strip()
# ---------------------------------------------------------------------------- #
# 备选 PDF 插件:pdfplumberlazy import,依赖缺失时不可用)
# ---------------------------------------------------------------------------- #
class PdfplumberTextExtractor:
"""pdfplumber 文本层提取(备选,对复杂排版更友好)"""
def extract_text(self, content: bytes) -> str:
import pdfplumber # type: ignore[import-untyped]
parts: list[str] = []
with pdfplumber.open(io.BytesIO(content)) as pdf:
for page in pdf.pages:
text = page.extract_text() or ""
if text:
parts.append(text)
return "\n".join(parts).strip()
# ---------------------------------------------------------------------------- #
# 默认 DOCX 插件:python-docx
# ---------------------------------------------------------------------------- #
class PythonDocxParser:
"""python-docx 段落提取(默认)"""
def extract_text(self, content: bytes) -> str:
from docx import Document # type: ignore[import-untyped]
document = Document(io.BytesIO(content))
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
return "\n".join(parts).strip()
# ---------------------------------------------------------------------------- #
# 默认 OCR 插件:rapidocr-onnxruntime + pypdfium2
# ---------------------------------------------------------------------------- #
class RapidocrOcrEngine:
"""rapidocr-onnxruntime OCR 引擎(默认)
流程:pypdfium2 渲染每页为 PIL Image → rapidocr 识别 → 拼接。
类级懒加载单例(_engine/_unavailable 为类属性,跨实例共享),
依赖缺失时降级返回空文本。
"""
from pypdf import PdfReader
reader = PdfReader(io.BytesIO(content))
parts: list[str] = []
for page in reader.pages:
text = page.extract_text() or ""
if text:
parts.append(text)
text_layer = "\n".join(parts).strip()
_engine: Any | None = None
_unavailable: bool = False
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
if self._unavailable:
return ""
# 1. 懒加载 OCR 引擎(类级单例,跨实例共享)
if self._engine is None:
try:
from rapidocr_onnxruntime import RapidOCR
RapidocrOcrEngine._engine = RapidOCR()
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
except Exception:
RapidocrOcrEngine._unavailable = True
logger.warning(
"OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True
)
return ""
# 2. 渲染并识别
try:
import pypdfium2 as pdfium
scale = max(1.0, dpi / 72.0)
pdf = pdfium.PdfDocument(io.BytesIO(content))
total = min(len(pdf), max(1, max_pages))
page_texts: list[str] = []
for i in range(total):
page = pdf[i]
pil_image = page.render(scale=scale).to_pil()
result, _ = self._engine(pil_image)
if result:
lines = [
item[1]
for item in result
if item and len(item) >= 2 and item[1]
]
if lines:
page_texts.append("\n".join(lines))
pdf.close()
return "\n".join(page_texts).strip()
except Exception as exc:
logger.warning(
"PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True
)
return ""
# ---------------------------------------------------------------------------- #
# 备选 OCR 插件:tesseractlazy import,依赖缺失时不可用)
# ---------------------------------------------------------------------------- #
class TesseractOcrEngine:
"""pytesseract + pdfium OCR(备选)
需要系统安装 tesseract 二进制与语言包。
类级 _unavailable 单例,跨实例共享。
"""
_unavailable: bool = False
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
if self._unavailable:
return ""
try:
import pypdfium2 as pdfium
import pytesseract # type: ignore[import-untyped]
from PIL import Image # type: ignore[import-untyped]
except Exception:
TesseractOcrEngine._unavailable = True
logger.warning("tesseract 依赖不可用,降级返回空文本", exc_info=True)
return ""
try:
scale = max(1.0, dpi / 72.0)
pdf = pdfium.PdfDocument(io.BytesIO(content))
total = min(len(pdf), max(1, max_pages))
page_texts: list[str] = []
for i in range(total):
page = pdf[i]
pil_image = page.render(scale=scale).to_pil()
text = pytesseract.image_to_string(pil_image, lang="chi_sim+eng")
if text:
page_texts.append(text)
pdf.close()
return "\n".join(page_texts).strip()
except Exception as exc:
logger.warning(
"tesseract OCR 失败,降级返回空文本", error=str(exc), exc_info=True
)
return ""
class NoopOcrEngine:
"""关闭 OCR 的占位插件(strategy=none 时使用)"""
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
return ""
# ---------------------------------------------------------------------------- #
# 插件注册表
# ---------------------------------------------------------------------------- #
_PDF_PLUGINS: dict[str, type[PdfTextExtractor]] = {
"pypdf": PypdfTextExtractor,
"pdfplumber": PdfplumberTextExtractor,
}
_DOCX_PLUGINS: dict[str, type[DocxParser]] = {
"python_docx": PythonDocxParser,
}
_OCR_PLUGINS: dict[str, type[OcrEngine]] = {
"rapidocr": RapidocrOcrEngine,
"tesseract": TesseractOcrEngine,
"none": NoopOcrEngine,
}
def list_pdf_plugins() -> list[str]:
return list(_PDF_PLUGINS.keys())
def list_docx_plugins() -> list[str]:
return list(_DOCX_PLUGINS.keys())
def list_ocr_plugins() -> list[str]:
return list(_OCR_PLUGINS.keys())
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
# 进程级插件单例缓存
_pdf_plugin_cache: dict[str, PdfTextExtractor] = {}
_docx_plugin_cache: dict[str, DocxParser] = {}
_ocr_plugin_cache: dict[str, OcrEngine] = {}
def _get_pdf_plugin() -> PdfTextExtractor:
rt = get_runtime_settings()
name = rt.parsers.pdf.plugin or "pypdf"
if name in _pdf_plugin_cache:
return _pdf_plugin_cache[name]
cls = _PDF_PLUGINS.get(name)
if cls is None:
logger.warning("未知 PDF 插件,降级到 pypdf", requested=name)
cls = PypdfTextExtractor
name = "pypdf"
instance = cls()
_pdf_plugin_cache[name] = instance
return instance
def _get_docx_plugin() -> DocxParser:
rt = get_runtime_settings()
name = rt.parsers.docx.plugin or "python_docx"
if name in _docx_plugin_cache:
return _docx_plugin_cache[name]
cls = _DOCX_PLUGINS.get(name)
if cls is None:
logger.warning("未知 DOCX 插件,降级到 python_docx", requested=name)
cls = PythonDocxParser
name = "python_docx"
instance = cls()
_docx_plugin_cache[name] = instance
return instance
def _get_ocr_plugin() -> OcrEngine:
rt = get_runtime_settings()
name = rt.parsers.ocr.plugin or "rapidocr"
if name in _ocr_plugin_cache:
return _ocr_plugin_cache[name]
cls = _OCR_PLUGINS.get(name)
if cls is None:
logger.warning("未知 OCR 插件,降级到 rapidocr", requested=name)
cls = RapidocrOcrEngine
name = "rapidocr"
instance = cls()
_ocr_plugin_cache[name] = instance
return instance
def invalidate_parser_plugin_cache(kind: str | None = None) -> None:
"""清除插件缓存(runtime_settings 更新后调用)"""
if kind is None or kind == "pdf":
_pdf_plugin_cache.clear()
if kind is None or kind == "docx":
_docx_plugin_cache.clear()
if kind is None or kind == "ocr":
_ocr_plugin_cache.clear()
# 重置 RapidocrOcrEngine 类级标记,允许重新初始化
RapidocrOcrEngine._engine = None
RapidocrOcrEngine._unavailable = False
TesseractOcrEngine._unavailable = False
# ---------------------------------------------------------------------------- #
# 顶层解析函数
# ---------------------------------------------------------------------------- #
def _parse_pdf(content: bytes) -> str:
"""PDF 解析:优先文本层插件;文本层为空时降级 OCR 插件"""
rt = get_runtime_settings()
pdf_plugin = _get_pdf_plugin()
try:
text_layer = pdf_plugin.extract_text(content)
except Exception as exc:
logger.warning(
"PDF 文本层提取失败,尝试 OCR 降级",
plugin=rt.parsers.pdf.plugin,
error=str(exc),
exc_info=True,
)
text_layer = ""
# 文本层非空:直接返回
if text_layer:
return text_layer
# 文本层为空 → 尝试 OCR 降级
# 文本层为空 → OCR 降级
if not settings.pdf_ocr_enabled:
return ""
ocr_text = _ocr_pdf(
content, max_pages=settings.pdf_ocr_max_pages, dpi=settings.pdf_ocr_dpi
ocr_plugin = _get_ocr_plugin()
return ocr_plugin.ocr_pdf(
content,
max_pages=rt.parsers.ocr.params.get("max_pages", settings.pdf_ocr_max_pages),
dpi=rt.parsers.ocr.params.get("dpi", settings.pdf_ocr_dpi),
)
return ocr_text
def _ocr_pdf(content: bytes, max_pages: int, dpi: int) -> str:
"""对扫描件 PDF 跑 OCR:渲染每页 → 识别 → 拼接
返回空字符串的场景:依赖未安装 / 渲染或识别异常 / 无识别结果。
任何异常仅告警不抛出,由上游按"无法提取文本"处理。
"""
global _ocr_engine, _ocr_unavailable
if _ocr_unavailable:
return ""
# 1. 懒加载 OCR 引擎
if _ocr_engine is None:
try:
from rapidocr_onnxruntime import RapidOCR
_ocr_engine = RapidOCR()
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
except Exception:
_ocr_unavailable = True
logger.warning("OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True)
return ""
# 2. 渲染并识别
try:
import pypdfium2 as pdfium
scale = max(1.0, dpi / 72.0)
pdf = pdfium.PdfDocument(io.BytesIO(content))
total = min(len(pdf), max(1, max_pages))
page_texts: list[str] = []
for i in range(total):
page = pdf[i]
pil_image = page.render(scale=scale).to_pil()
result, _ = _ocr_engine(pil_image)
if result:
# result: [[box, text, score], ...],按行拼接
lines = [
item[1] for item in result if item and len(item) >= 2 and item[1]
]
if lines:
page_texts.append("\n".join(lines))
pdf.close()
return "\n".join(page_texts).strip()
except Exception as exc:
logger.warning("PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True)
return ""
def _parse_docx(content: bytes) -> str:
"""DOCX 段落文本拼接(不含表格/页眉页脚)"""
from docx import Document # type: ignore[import-untyped]
document = Document(io.BytesIO(content))
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
return "\n".join(parts).strip()
"""DOCX 解析"""
return _get_docx_plugin().extract_text(content)
# 扩展名 → 解析函数映射(启动时构建,避免每次请求重复构造)
# 扩展名 → 解析函数映射
_PARSERS: dict[str, Callable[[bytes], str]] = {
".txt": _parse_text,
".md": _parse_text,
+13 -37
View File
@@ -6,10 +6,12 @@
内存注册表为主(记录 status/created_at/updated_at/result/error),
Redis 为持久镜像(key: ingest_task:{task_id}),每次状态迁移同步写入;
Redis 不可用或写入失败仅记录 warning,不影响任务执行。
文本去重委托给 app.core.dedup.get_dedup_strategy(),按 runtime_settings.dedup
选择策略(none/sha256/simhash)。
"""
import asyncio
import hashlib
import uuid
from datetime import UTC, datetime
from enum import StrEnum
@@ -18,6 +20,7 @@ from typing import Any
import structlog
from app.config import Settings
from app.core.dedup import DEDUP_KEY_PREFIX, get_dedup_strategy
from app.core.ingestion import Ingester, IngestionError
from app.models.document import DocumentInput
from app.services.redis import RedisCache
@@ -26,8 +29,6 @@ logger = structlog.get_logger()
# Redis 任务状态 key 前缀
REDIS_KEY_PREFIX = "ingest_task:"
# Redis 文本去重 key 前缀(value 为已入库文档的 IngestionResult JSON
DEDUP_KEY_PREFIX = "dedup:sha256:"
class IngestTaskStatus(StrEnum):
@@ -71,17 +72,17 @@ class IngestTaskManager:
async def submit(self, doc: DocumentInput) -> str:
"""登记入库任务并后台执行,立即返回 task_id
文本去重:基于 doc.text 的 sha256 在 Redis 中查重;命中则直接复用旧
IngestionResult(仅置 deduplicated=True),不重跑流水线;未命中走原
异步入库流程,完成后写入去重记录供后续命中复用。Redis 不可用时跳过
去重,按原流程执行,不影响主流程
文本去重:按 runtime_settings.dedup 选择策略(none/sha256/simhash);
命中则直接复用旧 IngestionResult(仅置 deduplicated=True),不重跑流水线;
未命中走原异步入库流程,完成后写入去重记录供后续命中复用。
Redis 不可用或策略=none 时跳过,按原流程执行
"""
task_id = uuid.uuid4().hex
now = _utc_now_iso()
text_hash = hashlib.sha256(doc.text.encode("utf-8")).hexdigest()
dedup = get_dedup_strategy(self._redis)
# 1. 去重命中:直接置 done,复用旧结果,不调 _run
dedup_record = await self._lookup_dedup(text_hash)
dedup_record = await dedup.lookup(doc.text)
if dedup_record is not None:
result_dict = dict(dedup_record)
result_dict["deduplicated"] = True
@@ -111,7 +112,7 @@ class IngestTaskManager:
"error": None,
}
self._schedule_mirror(task_id)
background = asyncio.create_task(self._run(task_id, doc, text_hash))
background = asyncio.create_task(self._run(task_id, doc, dedup))
self._background_tasks.add(background)
background.add_done_callback(self._background_tasks.discard)
logger.info("入库任务已登记", task_id=task_id, title=doc.title)
@@ -151,7 +152,7 @@ class IngestTaskManager:
raise TimeoutError(f"入库任务 {task_id}{timeout}s 内未进入终态")
await asyncio.sleep(0.01)
async def _run(self, task_id: str, doc: DocumentInput, text_hash: str) -> None:
async def _run(self, task_id: str, doc: DocumentInput, dedup: Any) -> None:
"""后台执行入库:并发限流 + 阶段状态推进 + 结果/错误落账 + 去重记录写入"""
async with self._semaphore:
try:
@@ -185,34 +186,9 @@ class IngestTaskManager:
result=result_dict,
)
self._schedule_mirror(task_id)
await self._record_dedup(text_hash, result_dict)
await dedup.record(doc.text, result_dict)
logger.info("入库任务完成", task_id=task_id)
async def _lookup_dedup(self, text_hash: str) -> dict[str, Any] | None:
"""查询文本去重记录;Redis 不可用或异常时降级为未命中"""
if self._redis is None:
return None
try:
return await self._redis.get_json(f"{DEDUP_KEY_PREFIX}{text_hash}")
except Exception:
logger.warning(
"去重记录查询失败,降级为未命中", text_hash=text_hash, exc_info=True
)
return None
async def _record_dedup(self, text_hash: str, result_dict: dict[str, Any]) -> None:
"""写入文本去重记录(含完整 IngestionResult),供后续命中复用;失败仅告警"""
if self._redis is None:
return
try:
await self._redis.set_json(
f"{DEDUP_KEY_PREFIX}{text_hash}",
result_dict,
ttl=self._settings.ingest_task_ttl_done,
)
except Exception:
logger.warning("去重记录写入失败", text_hash=text_hash, exc_info=True)
def _finish_failed(self, task_id: str, error: dict[str, Any]) -> None:
"""将任务置为 failed 并记录错误信息"""
self._tasks[task_id].update(
+14 -8
View File
@@ -1,6 +1,7 @@
"""query 解析与分类路由模块
分层 RAG 在线侧第一步:用 Ollama 小模型将用户 query 解析为结构化 JSON
分层 RAG 在线侧第一步:用 LLMOllama 或 OpenAI 兼容服务,由
runtime_settings.models.query 决定)将用户 query 解析为结构化 JSON
(命中类目+置信度、rewrite 后 query、关键词),再由纯函数做路由决策:
- 高置信且命中类目数 <= 上限 → 按类目过滤检索
- 低置信 / 解析失败 / 命中类目过多 → 全库兜底(不丢召回)
@@ -16,8 +17,8 @@ import structlog
from pydantic import BaseModel, Field, ValidationError
from app.config import settings
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory
from app.services.ollama import OllamaClient
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory, load_taxonomy
from app.services.llm import LLMClient, create_llm_client
from app.services.redis import RedisCache, get_cache
logger = structlog.get_logger()
@@ -106,17 +107,22 @@ def decide_route(parsed: ParsedQuery, threshold: float, max_categories: int) ->
class QueryParser:
"""query 解析器:调用 Ollama 小模型将 query 解析为结构化 JSON"""
"""query 解析器:调用 LLM 将 query 解析为结构化 JSON"""
def __init__(
self, ollama: OllamaClient, taxonomy: list[TaxonomyCategory], cache: RedisCache | None = None
self,
ollama: LLMClient | None = None,
taxonomy: list[TaxonomyCategory] | None = None,
cache: RedisCache | None = None,
) -> None:
self.ollama = ollama
self.taxonomy = taxonomy
# 默认按 runtime_settings.models.query 选择 LLM 实现;
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("query")
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
# 解析结果缓存,缺省用全局单例;RedisCache 全操作容错,缓存不可用时退化为无缓存行为
self.cache = cache if cache is not None else get_cache()
# 可作为路由命中类目的名字集合(uncategorized 不可作为路由命中类目)
self._routable_names = {c.name for c in taxonomy if c.name != UNCATEGORIZED}
self._routable_names = {c.name for c in self.taxonomy if c.name != UNCATEGORIZED}
async def parse(self, query: str) -> ParsedQuery:
"""调用 LLM 将 query 解析为结构化结果
+7 -4
View File
@@ -1,6 +1,7 @@
"""检索结果 AI 总结
对检索返回的 chunk 命中结果,调用 Ollama 本地模型生成一段针对用户 query 的总结回答。
对检索返回的 chunk 命中结果,调用 LLMOllama 或 OpenAI 兼容服务,由
runtime_settings.models.query 决定)生成一段针对用户 query 的总结回答。
仅基于检索结果内容,不编造未提及的信息。
"""
@@ -8,7 +9,7 @@ import structlog
from app.config import settings
from app.models.search import SearchHit
from app.services.ollama import OllamaClient
from app.services.llm import LLMClient, create_llm_client
logger = structlog.get_logger()
@@ -16,8 +17,10 @@ logger = structlog.get_logger()
class ResultSummarizer:
"""检索结果总结器"""
def __init__(self, ollama: OllamaClient | None = None) -> None:
self.ollama = ollama or OllamaClient()
def __init__(self, ollama: LLMClient | None = None) -> None:
# 默认按 runtime_settings.models.query 选择 LLM 实现;
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("query")
async def summarize(self, query: str, hits: list[SearchHit]) -> str:
"""对检索结果生成针对 query 的总结
+2 -2
View File
@@ -21,7 +21,7 @@ from app.core.result_summarizer import ResultSummarizer
from app.core.sparse import SparseEncoder
from app.models.knowledge import load_taxonomy
from app.models.search import ExtractedInfo, SearchHit, SearchRequest, SearchResponse
from app.services.ollama import OllamaClient
from app.services.llm import create_llm_client
from app.services.qdrant import (
COLLECTION_CHUNKS,
COLLECTION_L1,
@@ -52,7 +52,7 @@ class Retriever:
) -> None:
self.qdrant = qdrant or QdrantService()
self.query_parser = query_parser or QueryParser(
ollama=OllamaClient(),
ollama=create_llm_client("query"),
taxonomy=load_taxonomy(settings.taxonomy_path),
)
self.embedding = embedding or create_embedding_service()
+224
View File
@@ -0,0 +1,224 @@
"""运行时可调配置(runtime settings
与 `app.config.settings`(启动时从环境变量加载、不可变)互补:本模块管理运行时
可通过管理后台动态调整的子集,持久化到 JSON 文件,启动时加载覆盖到内存单例。
设计要点:
- RuntimeSettings 只包含「可运行时调整」的字段(模型/解析插件/去重等),
不包含敏感或启动期固定的字段(端口、数据库连接等)。
- 持久化路径默认 `data/runtime_settings.json`,可由 env `RUNTIME_SETTINGS_PATH` 覆盖。
- 加载失败/文件缺失时回退到默认值,不阻塞启动。
- 写入采用「先临时文件后 rename」原子替换,避免半写损坏。
- 全模块只通过 `get_runtime_settings()` 访问单例,避免直接读 JSON。
"""
from __future__ import annotations
import json
import os
import tempfile
from pathlib import Path
from threading import RLock
from typing import Any, Literal
import structlog
from pydantic import BaseModel, Field
from app.config import settings
logger = structlog.get_logger()
# 默认持久化路径(相对工作目录);可由 env 覆盖
DEFAULT_RUNTIME_SETTINGS_PATH = "./data/runtime_settings.json"
_lock = RLock()
# ---------------------------------------------------------------------------- #
# 模型配置(文档总结 / 查询 / 分类 各一份独立配置)
# ---------------------------------------------------------------------------- #
class LlmProviderConfig(BaseModel):
"""单个用途的 LLM 提供方配置"""
provider: Literal["ollama", "openai_compatible"] = Field(
default="ollama", description="提供方:ollama 走 Ollama HTTPopenai_compatible 走 OpenAI 兼容 chat/completions"
)
base_url: str = Field(default="", description="服务地址;空则 ollama 用 settings.ollama_base_urlopenai_compatible 用 settings.openai_base_url")
api_key: str = Field(default="", description="API Key(仅 openai_compatible 需要;ollama 忽略)")
model: str = Field(default="", description="模型名;空则 ollama 用 settings.ollama_modelopenai_compatible 用 settings.embedding_model 同级(如 gpt-4o-mini")
timeout: float = Field(default=120.0, description="请求超时秒数")
temperature: float = Field(default=0.3, description="采样温度(0~2")
class ModelSettings(BaseModel):
"""模型相关运行时配置:三种用途独立配置"""
summarize: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档三级总结")
query: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="检索链路 query 解析与结果总结")
classify: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档分类判定")
# ---------------------------------------------------------------------------- #
# 解析插件配置
# ---------------------------------------------------------------------------- #
class PluginConfig(BaseModel):
"""单个解析插件的选择与参数"""
plugin: str = Field(default="", description="插件名;空则用默认")
params: dict[str, Any] = Field(default_factory=dict, description="插件参数(透传给插件实现)")
class ParserSettings(BaseModel):
"""解析插件运行时配置"""
ocr: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="rapidocr"), description="OCR 插件")
pdf: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="pypdf"), description="PDF 文本层提取插件")
docx: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="python_docx"), description="DOCX 解析插件")
# ---------------------------------------------------------------------------- #
# 去重策略配置
# ---------------------------------------------------------------------------- #
class DedupSettings(BaseModel):
"""文本去重策略运行时配置"""
strategy: Literal["none", "sha256", "simhash"] = Field(
default="sha256", description="去重策略:none 关闭;sha256 精确匹配;simhash 近似匹配"
)
simhash_threshold: int = Field(default=3, description="simhash 海明距离阈值(仅 strategy=simhash 生效,0~64")
ttl_seconds: int = Field(default=86400, description="去重记录 Redis 保留秒数")
# ---------------------------------------------------------------------------- #
# 顶层 RuntimeSettings
# ---------------------------------------------------------------------------- #
class RuntimeSettings(BaseModel):
"""运行时可调配置顶层模型"""
models: ModelSettings = Field(default_factory=ModelSettings)
parsers: ParserSettings = Field(default_factory=ParserSettings)
dedup: DedupSettings = Field(default_factory=DedupSettings)
# ---------------------------------------------------------------------------- #
# 单例 + 持久化
# ---------------------------------------------------------------------------- #
_runtime_settings: RuntimeSettings | None = None
def _resolve_path() -> Path:
"""解析持久化文件路径:env RUNTIME_SETTINGS_PATH > settings 自定义 > 默认"""
path_str = os.environ.get("RUNTIME_SETTINGS_PATH", "") or getattr(settings, "runtime_settings_path", "") or DEFAULT_RUNTIME_SETTINGS_PATH
return Path(path_str).expanduser().resolve()
def _default_with_env_fallback() -> RuntimeSettings:
"""构造默认 RuntimeSettings,并把启动 env 中已有的模型相关字段填充进去
这样首次启动(无持久化文件)时,UI 显示的不是空字符串而是 env 当前值。
"""
cfg = RuntimeSettings()
# 模型默认值沿用 env
cfg.models.summarize.base_url = settings.ollama_base_url
cfg.models.summarize.model = settings.ollama_model
cfg.models.query.base_url = settings.ollama_base_url
cfg.models.query.model = settings.ollama_model
cfg.models.classify.base_url = settings.ollama_base_url
cfg.models.classify.model = settings.ollama_model
# 若 env 提供了 OpenAI key/url,预填到 openai_compatible 字段方便切换
if settings.openai_api_key:
for usage in ("summarize", "query", "classify"):
getattr(cfg.models, usage).api_key = settings.openai_api_key
getattr(cfg.models, usage).base_url = settings.openai_base_url if settings.openai_base_url else getattr(cfg.models, usage).base_url
return cfg
def load_runtime_settings() -> RuntimeSettings:
"""从磁盘加载 RuntimeSettings;文件缺失或损坏时回退到默认值(带 env 兜底)"""
path = _resolve_path()
try:
if path.exists():
data = json.loads(path.read_text(encoding="utf-8"))
return RuntimeSettings.model_validate(data)
except Exception:
logger.warning("RuntimeSettings 加载失败,回退默认值", path=str(path), exc_info=True)
return _default_with_env_fallback()
def save_runtime_settings(cfg: RuntimeSettings) -> None:
"""原子写入 RuntimeSettings 到磁盘(先临时文件后 rename)"""
path = _resolve_path()
path.parent.mkdir(parents=True, exist_ok=True)
data = cfg.model_dump(mode="json")
# 写到同目录临时文件再 rename,避免半写损坏
fd, tmp_path = tempfile.mkstemp(prefix=".runtime_settings.", suffix=".tmp", dir=str(path.parent))
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
os.replace(tmp_path, path)
except Exception:
try:
os.unlink(tmp_path)
except OSError:
pass
raise
def get_runtime_settings() -> RuntimeSettings:
"""获取 RuntimeSettings 单例(首次调用时加载)"""
global _runtime_settings
with _lock:
if _runtime_settings is None:
_runtime_settings = load_runtime_settings()
return _runtime_settings
def update_runtime_settings(patch: dict[str, Any]) -> RuntimeSettings:
"""以 patch 字典更新 RuntimeSettings 并持久化
支持部分更新(顶层键可缺失,缺省保留原值)。例如:
update_runtime_settings({"models": {"summarize": {"model": "qwen2.5:3b"}}})
"""
with _lock:
current = get_runtime_settings()
merged = current.model_dump(mode="json")
_deep_merge(merged, patch)
new_cfg = RuntimeSettings.model_validate(merged)
save_runtime_settings(new_cfg)
# 替换单例后立即返回,新值对所有后续读取生效
global _runtime_settings
_runtime_settings = new_cfg
logger.info("RuntimeSettings 已更新并持久化", path=str(_resolve_path()))
return new_cfg
def _deep_merge(target: dict[str, Any], patch: dict[str, Any]) -> None:
"""递归把 patch 合并到 target(同 key 字典则递归,否则覆盖)"""
for k, v in patch.items():
if k in target and isinstance(target[k], dict) and isinstance(v, dict):
_deep_merge(target[k], v)
else:
target[k] = v
def reload_runtime_settings() -> RuntimeSettings:
"""强制从磁盘重新加载(管理后台触发)"""
with _lock:
global _runtime_settings
_runtime_settings = load_runtime_settings()
return _runtime_settings
def reset_runtime_settings() -> RuntimeSettings:
"""重置为默认值并持久化(管理后台触发)"""
with _lock:
new_cfg = _default_with_env_fallback()
save_runtime_settings(new_cfg)
global _runtime_settings
_runtime_settings = new_cfg
return new_cfg
+7 -4
View File
@@ -1,6 +1,7 @@
"""文档三级总结模块
通过 Ollama 本地小模型对文档进行分级总结:
通过 LLMOllama 或 OpenAI 兼容服务,由 runtime_settings.models.summarize 决定)
对文档进行分级总结:
- L1: 总结(一句话高度概括)
- L2: 大纲(主要章节和关键主题)
- L3: 内容大纲(每个章节的详细内容摘要)
@@ -12,7 +13,7 @@ import structlog
from app.core.headings import parse_headings, render_outline
from app.models.document import DocumentSummary, SummaryLevel
from app.services.ollama import OllamaClient
from app.services.llm import LLMClient, create_llm_client
logger = structlog.get_logger()
@@ -23,8 +24,10 @@ MIN_TEXT_LENGTH_FOR_L3 = 500
class Summarizer:
"""文档三级总结器"""
def __init__(self, ollama: OllamaClient | None = None) -> None:
self.ollama = ollama or OllamaClient()
def __init__(self, ollama: LLMClient | None = None) -> None:
# 默认按 runtime_settings 选择 LLM 实现(Ollama 或 OpenAI 兼容);
# 测试可通过 ollama 参数注入替身。
self.ollama = ollama or create_llm_client("summarize")
async def summarize(self, text: str, *, title: str = "") -> DocumentSummary:
"""对文档文本进行三级总结
+50 -4
View File
@@ -5,13 +5,14 @@ from pathlib import Path
import structlog
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import FileResponse, JSONResponse
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
from app.api.response import ApiError, error
from app.api.v1.auth import router as auth_router
from app.api.v1.document import router as document_router
from app.api.v1.knowledge import router as knowledge_router
from app.api.v1.search import router as search_router
from app.api.v1.settings import router as settings_router
from app.config import settings
from app.core.auth import ensure_default_admin
from app.services.qdrant import QdrantService
@@ -53,6 +54,7 @@ app.include_router(auth_router)
app.include_router(search_router)
app.include_router(document_router)
app.include_router(knowledge_router)
app.include_router(settings_router)
@app.exception_handler(ApiError)
@@ -84,10 +86,54 @@ async def health() -> dict[str, str]:
return {"status": "ok"}
_ADMIN_HTML = Path(__file__).resolve().parent / "static" / "admin.html"
_STATIC_DIR = Path(__file__).resolve().parent / "static"
_ADMIN_HTML = _STATIC_DIR / "admin.html"
_ADMIN_SPA_DIR = _STATIC_DIR / "admin"
def _spa_index() -> Path | None:
"""SPA 入口文件路径;不存在返回 None(回退旧 admin.html"""
index = _ADMIN_SPA_DIR / "index.html"
return index if index.is_file() else None
@app.get("/admin", include_in_schema=False)
async def admin_page() -> FileResponse:
"""管理后台单页(单文件静态 HTML,零外部依赖)"""
async def admin_page():
"""管理后台:优先服务 Vue SPA,回退到旧单文件 admin.html"""
spa = _spa_index()
if spa is not None:
return FileResponse(spa, media_type="text/html")
return FileResponse(_ADMIN_HTML, media_type="text/html")
@app.get("/admin/", include_in_schema=False)
async def admin_page_trailing_slash():
"""带斜杠的 /admin/ 重定向到 /admin"""
return RedirectResponse(url="/admin", status_code=307)
@app.get("/admin/{rest:path}", include_in_schema=False)
async def admin_spa(rest: str):
"""SPA 静态资源与客户端路由兜底
- 真实文件(assets/*.js, *.css 等)→ 直接返回
- 其余路径(/overview, /documents 等客户端路由)→ 返回 index.html
- SPA 目录不存在时 → 404(旧 admin.html 无子路由需求)
"""
spa_dir = _ADMIN_SPA_DIR
if not spa_dir.is_dir():
return JSONResponse(
status_code=404, content={"code": 1002, "message": "Not found"}
)
# 安全校验:防止路径越界
file_path = (spa_dir / rest).resolve()
try:
file_path.relative_to(spa_dir.resolve())
except ValueError:
return FileResponse(spa_dir / "index.html", media_type="text/html")
if file_path.is_file():
return FileResponse(file_path)
# SPA 客户端路由兜底
return FileResponse(spa_dir / "index.html", media_type="text/html")
+179
View File
@@ -0,0 +1,179 @@
"""LLM 客户端抽象层
提供统一的 `generate(prompt, json_mode)` 接口,底层实现可切换:
- OllamaLLMClient:走 Ollama HTTP /api/generate(本地或远程 Ollama 服务)
- OpenAICompatibleLLMClient:走 OpenAI 兼容 /v1/chat/completionsOpenAI / DeepSeek / 智谱 / Qwen API 等)
工厂 `create_llm_client(purpose)` 根据 runtime_settings 选择实现与参数。
所有客户端共享同一接口,业务侧无需感知底层协议差异。
"""
from __future__ import annotations
from typing import Literal, Protocol, runtime_checkable
import httpx
import structlog
from app.config import settings
from app.core.runtime_settings import LlmProviderConfig, get_runtime_settings
from app.services.ollama import OllamaClient
logger = structlog.get_logger()
LlmPurpose = Literal["summarize", "query", "classify"]
@runtime_checkable
class LLMClient(Protocol):
"""LLM 客户端统一接口"""
async def generate(self, prompt: str, json_mode: bool = False) -> str:
"""生成文本
Args:
prompt: 输入提示词
json_mode: True 时约束输出为 JSON(不支持时降级为普通生成)
Returns:
生成的文本内容
"""
...
async def is_available(self) -> bool:
"""检查服务是否可用"""
...
class OllamaLLMClient:
"""Ollama HTTP API 客户端(包装现有 OllamaClient,便于统一接口)"""
def __init__(self, base_url: str, model: str, timeout: float = 120.0) -> None:
self._inner = OllamaClient(base_url=base_url, model=model, timeout=timeout)
async def generate(self, prompt: str, json_mode: bool = False) -> str:
return await self._inner.generate(prompt, json_mode=json_mode)
async def is_available(self) -> bool:
return await self._inner.is_available()
class OpenAICompatibleLLMClient:
"""OpenAI 兼容 chat/completions 客户端
适用于 OpenAI 官方 API、DeepSeek、智谱 ChatGLM、Qwen DashScope、Moonshot 等
所有兼容 OpenAI /v1/chat/completions 协议的服务。
"""
def __init__(self, base_url: str, api_key: str, model: str, timeout: float = 120.0, temperature: float = 0.3) -> None:
self.base_url = base_url.rstrip("/")
self.api_key = api_key
self.model = model
self.timeout = timeout
self.temperature = temperature
async def generate(self, prompt: str, json_mode: bool = False) -> str:
url = f"{self.base_url}/chat/completions"
payload: dict = {
"model": self.model,
"messages": [{"role": "user", "content": prompt}],
"temperature": self.temperature,
"stream": False,
}
if json_mode:
payload["response_format"] = {"type": "json_object"}
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
async with httpx.AsyncClient(timeout=self.timeout) as client:
resp = await client.post(url, json=payload, headers=headers)
resp.raise_for_status()
data = resp.json()
# OpenAI 标准响应结构:choices[0].message.content
choices = data.get("choices") or []
if not choices:
logger.warning("OpenAI 兼容响应无 choices", model=self.model, raw_keys=list(data.keys()))
return ""
message = choices[0].get("message") or {}
content = message.get("content") or ""
logger.debug("OpenAI 兼容生成完成", model=self.model, output_length=len(content))
return content
async def is_available(self) -> bool:
"""简单探测:调 /models 列表接口(多数 OpenAI 兼容服务支持)"""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(
f"{self.base_url}/models",
headers={"Authorization": f"Bearer {self.api_key}"},
)
return resp.status_code == 200
except httpx.HTTPError:
return False
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
# 进程级客户端缓存(避免每请求新建 httpx client
_client_cache: dict[str, LLMClient] = {}
def _resolve_provider_config(purpose: LlmPurpose) -> LlmProviderConfig:
"""从 runtime_settings 取指定用途的 LLM 配置"""
rt = get_runtime_settings()
return getattr(rt.models, purpose)
def _build_client(purpose: LlmPurpose) -> LLMClient:
"""按 runtime_settings 构造 LLM 客户端"""
cfg = _resolve_provider_config(purpose)
if cfg.provider == "ollama":
base_url = cfg.base_url or settings.ollama_base_url
model = cfg.model or settings.ollama_model
return OllamaLLMClient(base_url=base_url, model=model, timeout=cfg.timeout)
if cfg.provider == "openai_compatible":
base_url = cfg.base_url or settings.openai_base_url
api_key = cfg.api_key or settings.openai_api_key
# openai_compatible 默认模型:若 cfg.model 为空,退化到 OpenAI 通用 chat 模型
model = cfg.model or "gpt-4o-mini"
if not api_key:
logger.warning("OpenAI 兼容 provider 缺少 api_key,调用大概率会失败", purpose=purpose)
return OpenAICompatibleLLMClient(
base_url=base_url, api_key=api_key, model=model, timeout=cfg.timeout, temperature=cfg.temperature
)
raise ValueError(f"未知 LLM provider: {cfg.provider}")
def create_llm_client(purpose: LlmPurpose, *, use_cache: bool = True) -> LLMClient:
"""创建指定用途的 LLM 客户端
Args:
purpose: 用途(summarize / query / classify
use_cache: True 时复用进程级客户端单例(默认);False 每次新建(测试用)
Returns:
LLMClient 实例
"""
if not use_cache:
return _build_client(purpose)
cache_key = f"{purpose}"
if cache_key not in _client_cache:
_client_cache[cache_key] = _build_client(purpose)
return _client_cache[cache_key]
def invalidate_llm_client_cache(purpose: LlmPurpose | None = None) -> None:
"""清除客户端缓存(runtime_settings 更新后调用,确保后续读取新配置)"""
if purpose is None:
_client_cache.clear()
else:
_client_cache.pop(purpose, None)
+22 -5
View File
@@ -10,6 +10,8 @@ services:
- QDRANT_PORT=6333
- REDIS_URL=redis://redis:6379/0
- OLLAMA_BASE_URL=http://ollama:11434
# 针对 16 线程 / 61GB 内存的 NAS 调优:放宽入库并发
- INGEST_MAX_CONCURRENCY=${INGEST_MAX_CONCURRENCY:-4}
env_file:
- .env
depends_on:
@@ -21,6 +23,7 @@ services:
condition: service_started
volumes:
- ${NAS_DATA_DIR:-./data}/logs:/app/logs
- ${NAS_DATA_DIR:-./data}/uploads:/app/uploads
networks:
- qmdsearch
@@ -65,11 +68,25 @@ services:
- "${OLLAMA_PORT:-11434}:11434"
volumes:
- ${NAS_DATA_DIR:-./data}/ollama:/root/.ollama
# 首次启动后需手动拉取模型:
# docker exec qmdsearch-ollama ollama pull qwen2.5:1.5b
# 或取消下方 entrypoint 注释以自动拉取(需等待下载完成)
# entrypoint: /bin/bash
# command: -c "ollama serve & sleep 5 && ollama pull qwen2.5:1.5b && wait"
environment:
# 针对 Ryzen 9 7940HS16 线程)的 CPU 推理调优:
# 并行推理任务数、常驻模型数、单请求线程上限、KV 缓存量化以省内存
- OLLAMA_NUM_PARALLEL=4
- OLLAMA_MAX_LOADED_MODELS=2
- OLLAMA_NUM_THREADS=16
- OLLAMA_KV_CACHE_TYPE=q8_0
# 首次启动自动拉取所需模型(qwen2.5:1.5b 总结 + bge-m3 嵌入),
# 下载完成后转交常驻 ollama serve。已存在时仅做健康检查。
entrypoint: /bin/bash
command:
- -c
- |
ollama serve &
SERVE_PID=$$!
sleep 6
ollama pull qwen2.5:1.5b
ollama pull bge-m3
wait $$SERVE_PID
networks:
- qmdsearch
+6
View File
@@ -0,0 +1,6 @@
node_modules
dist
dist-ssr
*.local
.DS_Store
.vite
+12
View File
@@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>QMDSearch 知识库管理后台</title>
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/main.js"></script>
</body>
</html>
+15
View File
@@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "Bundler",
"jsx": "preserve",
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"]
},
"checkJs": false
},
"include": ["src/**/*.js", "src/**/*.vue"],
"exclude": ["node_modules", "dist"]
}
+1871
View File
File diff suppressed because it is too large Load Diff
+24
View File
@@ -0,0 +1,24 @@
{
"name": "qmdsearch-frontend",
"version": "1.0.0",
"private": true,
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview"
},
"dependencies": {
"@ant-design/icons-vue": "^7.0.1",
"ant-design-vue": "^4.2.6",
"axios": "^1.7.9",
"dayjs": "^1.11.13",
"pinia": "^2.3.0",
"vue": "^3.5.13",
"vue-router": "^4.5.0"
},
"devDependencies": {
"@vitejs/plugin-vue": "^5.2.1",
"vite": "^5.4.11"
}
}
+41
View File
@@ -0,0 +1,41 @@
<script setup>
import { RouterView } from 'vue-router'
import { ConfigProvider, theme as antdTheme } from 'ant-design-vue'
const themeConfig = {
algorithm: antdTheme.defaultAlgorithm,
token: {
colorPrimary: '#1677ff',
colorInfo: '#1677ff',
colorSuccess: '#52c41a',
colorWarning: '#faad14',
colorError: '#ff4d4f',
borderRadius: 6,
fontSize: 14,
fontFamily:
"-apple-system, BlinkMacSystemFont, 'PingFang SC', 'Microsoft YaHei', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif"
},
components: {
Layout: {
headerBg: '#ffffff',
headerPadding: '0 16px',
headerHeight: 52,
siderBg: '#001529'
},
Menu: {
darkItemBg: '#001529',
darkSubMenuItemBg: '#000c17'
}
}
}
</script>
<template>
<ConfigProvider :theme="themeConfig" :locale="undefined">
<RouterView />
</ConfigProvider>
</template>
<style scoped>
/* 全局根容器无额外样式,由各布局/页面自行处理 */
</style>
+19
View File
@@ -0,0 +1,19 @@
import http from './client'
/**
* 用户名密码登录
* @param {string} username
* @param {string} password
* @returns {Promise<{access_token:string, expires_in:number, user:{username:string, role:string, created_at:string}}>}
*/
export function login(username, password) {
return http.post('/api/v1/auth/login', { username, password })
}
/**
* 获取当前登录用户信息
* @returns {Promise<{username:string, role:string, created_at:string}>}
*/
export function me() {
return http.get('/api/v1/auth/me')
}
+108
View File
@@ -0,0 +1,108 @@
import axios from 'axios'
import { message } from 'ant-design-vue'
const TOKEN_STORAGE_KEY = 'qmd_token'
/** 认证相关错误码:触发清 token + 跳登录 */
const AUTH_ERROR_CODES = new Set([1003, 1005])
const httpClient = axios.create({
// 不设 baseURL,使用相对路径,由 vite proxy / nginx 转发
timeout: 60000,
headers: {
'Content-Type': 'application/json'
}
})
// 请求拦截器:注入 Bearer token
httpClient.interceptors.request.use((config) => {
const token = localStorage.getItem(TOKEN_STORAGE_KEY) || ''
if (token) {
config.headers = config.headers || {}
config.headers.Authorization = `Bearer ${token}`
}
return config
})
let unauthorizedHandler = null
/**
* 注册 401 / 认证错误处理回调(由 router/store 注入,避免循环依赖)
* @param {() => void} handler
*/
export function setUnauthorizedHandler(handler) {
unauthorizedHandler = handler
}
function triggerUnauthorized() {
localStorage.removeItem(TOKEN_STORAGE_KEY)
if (typeof unauthorizedHandler === 'function') {
unauthorizedHandler()
}
}
// 响应拦截器:统一处理 code !== 0 与 401
httpClient.interceptors.response.use(
(response) => {
const body = response.data
if (body && typeof body === 'object' && 'code' in body) {
if (body.code === 0) {
return body.data
}
// 业务错误
if (AUTH_ERROR_CODES.has(body.code)) {
triggerUnauthorized()
}
const err = new Error(body.message || '请求失败')
err.code = body.code
err.message = body.message || '请求失败'
return Promise.reject(err)
}
// 非标准结构,原样返回
return body
},
(error) => {
const status = error?.response?.status
if (status === 401) {
triggerUnauthorized()
const err = new Error('未认证或登录已过期,请重新登录')
err.code = 1003
return Promise.reject(err)
}
// 后端返回了 body 但 HTTP 错误
const body = error?.response?.data
if (body && typeof body === 'object' && 'code' in body) {
if (AUTH_ERROR_CODES.has(body.code)) {
triggerUnauthorized()
}
const err = new Error(body.message || `请求失败 (HTTP ${status ?? '?'})`)
err.code = body.code
return Promise.reject(err)
}
const err = new Error(error?.message || '网络请求失败')
err.code = `HTTP_${status ?? 'NETWORK'}`
return Promise.reject(err)
}
)
/**
* 统一发起请求,捕获异常并弹出 antd message
* @param {() => Promise<any>} fn
* @param {{ silent?: boolean, errorText?: string }} [options]
* @returns {Promise<any>}
*/
export async function callApi(fn, options = {}) {
const { silent = false, errorText = '操作失败' } = options
try {
return await fn()
} catch (err) {
const text = err?.message || errorText
if (!silent) {
message.error(text)
}
throw err
}
}
export { TOKEN_STORAGE_KEY }
export default httpClient
+62
View File
@@ -0,0 +1,62 @@
import http from './client'
/**
* 分页列出文档
* @param {number} [limit=20]
* @param {string|null} [offset=null]
* @returns {Promise<{items: Array, next_offset: string|null}>}
*/
export function list(limit = 20, offset = null) {
const params = { limit }
if (offset !== null && offset !== undefined && offset !== '') {
params.offset = offset
}
return http.get('/api/v1/documents', { params })
}
/**
* 获取文档详情
* @param {string} docId
* @returns {Promise<object>}
*/
export function detail(docId) {
return http.get(`/api/v1/documents/${encodeURIComponent(docId)}`)
}
/**
* 删除文档(幂等)
* @param {string} docId
* @returns {Promise<{doc_id:string, deleted: object, deleted_total:number}>}
*/
export function remove(docId) {
return http.delete(`/api/v1/documents/${encodeURIComponent(docId)}`)
}
/**
* JSON 文本入库(异步)
* @param {{title:string, source?:string, text:string, metadata?:object}} payload
* @returns {Promise<{task_id:string, status:string}>}
*/
export function ingest(payload) {
return http.post('/api/v1/documents', payload)
}
/**
* multipart 文件上传入库
* @param {FormData} formData
* @returns {Promise<{task_id:string, status:string, saved_path?:string}>}
*/
export function upload(formData) {
return http.post('/api/v1/documents/upload', formData, {
headers: { 'Content-Type': 'multipart/form-data' }
})
}
/**
* 查询入库任务状态
* @param {string} taskId
* @returns {Promise<object>}
*/
export function taskStatus(taskId) {
return http.get(`/api/v1/documents/tasks/${encodeURIComponent(taskId)}`)
}
+17
View File
@@ -0,0 +1,17 @@
import http from './client'
/**
* 获取知识分类类目集
* @returns {Promise<{categories: Array<{name:string, description:string}>, count: number}>}
*/
export function categories() {
return http.get('/api/v1/knowledge/categories')
}
/**
* 获取知识库统计(四层点数 + 类目分布 + uncategorized 数)
* @returns {Promise<{collections: {doc_l1:number, doc_l2:number, doc_l3:number, chunks:number}, documents_total:number, uncategorized_count:number, categories: Record<string, number>}>}
*/
export function stats() {
return http.get('/api/v1/knowledge/stats')
}
+10
View File
@@ -0,0 +1,10 @@
import http from './client'
/**
* 分层检索
* @param {{query:string, top_k?:number, summarize?:boolean}} payload
* @returns {Promise<object>}
*/
export function search(payload) {
return http.post('/api/v1/search', payload)
}
+34
View File
@@ -0,0 +1,34 @@
import http from './client'
/**
* 获取当前 RuntimeSettings
* @returns {Promise<object>}
*/
export function get() {
return http.get('/api/v1/settings')
}
/**
* 部分更新 RuntimeSettings(仅 admin
* @param {{models?:object, parsers?:object, dedup?:object}} payload
* @returns {Promise<object>}
*/
export function update(payload) {
return http.put('/api/v1/settings', payload)
}
/**
* 获取可选项 schema
* @returns {Promise<{llm_providers:string[], pdf_plugins:string[], docx_plugins:string[], ocr_plugins:string[], dedup_strategies:string[]}>}
*/
export function schema() {
return http.get('/api/v1/settings/schema')
}
/**
* 重置为默认值(仅 admin
* @returns {Promise<object>}
*/
export function reset() {
return http.post('/api/v1/settings/reset')
}
@@ -0,0 +1,93 @@
import { onBeforeUnmount, reactive, ref } from 'vue'
import { taskStatus } from '@/api/documents'
import {
INGEST_POLL_INTERVAL_MS,
INGEST_POLL_MAX_ATTEMPTS,
INGEST_TERMINAL_STATUS
} from '@/constants/ingest'
/**
* 入库任务轮询 composable
*
* 调用 startPolling(taskId) 启动轮询;到达 done/failed/超时/出错 时自动停止并
* 写入 state。组件卸载时自动清理定时器。
*
* @returns {{
* state: { taskId: string|null, task: object|null, status: string|null, error: object|null, isTimeout: boolean, isPolling: boolean },
* startPolling: (taskId: string) => void,
* stopPolling: () => void
* }}
*/
export function useIngestPolling() {
const state = reactive({
taskId: null,
task: null,
status: null,
error: null,
isTimeout: false,
isPolling: false
})
const timerRef = ref(null)
let attempts = 0
function stopPolling() {
if (timerRef.value !== null) {
clearInterval(timerRef.value)
timerRef.value = null
}
state.isPolling = false
}
function reset() {
stopPolling()
state.taskId = null
state.task = null
state.status = null
state.error = null
state.isTimeout = false
state.isPolling = false
attempts = 0
}
/**
* 启动轮询
* @param {string} taskId
*/
function startPolling(taskId) {
reset()
state.taskId = taskId
state.isPolling = true
attempts = 0
timerRef.value = setInterval(async () => {
attempts += 1
if (attempts > INGEST_POLL_MAX_ATTEMPTS) {
stopPolling()
state.isTimeout = true
return
}
try {
const task = await taskStatus(taskId)
state.task = task
state.status = task?.status || null
if (INGEST_TERMINAL_STATUS.includes(task?.status)) {
stopPolling()
}
} catch (err) {
state.error = err
stopPolling()
}
}, INGEST_POLL_INTERVAL_MS)
}
onBeforeUnmount(() => {
stopPolling()
})
return {
state,
startPolling,
stopPolling
}
}
+32
View File
@@ -0,0 +1,32 @@
/**
* 入库任务状态映射:后端 status → 中文展示
*/
export const INGEST_STATUS_TEXT = Object.freeze({
pending: '排队中',
summarizing: '总结中',
classifying: '分类中',
embedding: '向量化中',
writing: '写入中',
done: '完成',
failed: '失败'
})
/** 轮询间隔(毫秒) */
export const INGEST_POLL_INTERVAL_MS = 2000
/** 轮询最大次数:150 次 × 2s = 5 分钟超时 */
export const INGEST_POLL_MAX_ATTEMPTS = 150
/** 入库状态徽标颜色映射(antd Badge / Tag 状态) */
export const INGEST_STATUS_COLOR = Object.freeze({
pending: 'default',
summarizing: 'processing',
classifying: 'processing',
embedding: 'processing',
writing: 'processing',
done: 'success',
failed: 'error'
})
/** 终态集合:到达这些状态后停止轮询 */
export const INGEST_TERMINAL_STATUS = Object.freeze(['done', 'failed'])
+232
View File
@@ -0,0 +1,232 @@
<script setup>
import { computed, h, ref } from 'vue'
import { useRoute, useRouter, RouterView } from 'vue-router'
import { storeToRefs } from 'pinia'
import { Modal } from 'ant-design-vue'
import {
DashboardOutlined,
FileTextOutlined,
UploadOutlined,
SearchOutlined,
AppstoreOutlined,
SettingOutlined,
MenuFoldOutlined,
MenuUnfoldOutlined,
LogoutOutlined,
DatabaseOutlined
} from '@ant-design/icons-vue'
import { useAuthStore } from '@/stores/useAuthStore'
const route = useRoute()
const router = useRouter()
const authStore = useAuthStore()
const { user, displayName } = storeToRefs(authStore)
const collapsed = ref(false)
const selectedKeys = computed(() => {
return [route.name ? String(route.name) : '']
})
const openKeys = ref(['main'])
const menuItems = [
{ key: 'overview', icon: () => h(DashboardOutlined), label: '概览' },
{ key: 'documents', icon: () => h(FileTextOutlined), label: '文档管理' },
{ key: 'ingest', icon: () => h(UploadOutlined), label: '文档入库' },
{ key: 'search', icon: () => h(SearchOutlined), label: '检索测试台' },
{ key: 'categories', icon: () => h(AppstoreOutlined), label: '类目列表' },
{ key: 'settings', icon: () => h(SettingOutlined), label: '设置' }
]
function handleMenuClick({ key }) {
if (key && key !== route.name) {
router.push({ name: key })
}
}
function handleLogout() {
Modal.confirm({
title: '确认登出',
content: '确定要退出登录吗?',
okText: '登出',
cancelText: '取消',
onOk() {
authStore.logout()
router.replace({ name: 'login' })
}
})
}
</script>
<template>
<a-layout class="main-layout">
<a-layout-sider
v-model:collapsed="collapsed"
collapsible
:trigger="null"
breakpoint="lg"
class="main-layout__sider"
>
<div class="main-layout__logo">
<span class="main-layout__logo-icon">
<DatabaseOutlined />
</span>
<span v-if="!collapsed" class="main-layout__logo-text">QMDSearch</span>
</div>
<a-menu
theme="dark"
mode="inline"
:selected-keys="selectedKeys"
:open-keys="openKeys"
:items="menuItems"
@click="handleMenuClick"
/>
</a-layout-sider>
<a-layout>
<a-layout-header class="main-layout__header">
<a-button
type="text"
class="main-layout__collapse"
@click="collapsed = !collapsed"
>
<MenuUnfoldOutlined v-if="collapsed" />
<MenuFoldOutlined v-else />
</a-button>
<div class="main-layout__title">知识库管理后台</div>
<div class="main-layout__user">
<span class="main-layout__user-name">{{ displayName }}</span>
<a-divider type="vertical" />
<a-button
type="text"
size="small"
class="main-layout__logout"
@click="handleLogout"
>
<template #icon><LogoutOutlined /></template>
登出
</a-button>
</div>
</a-layout-header>
<a-layout-content class="main-layout__content">
<RouterView />
</a-layout-content>
<a-layout-footer class="main-layout__footer">
QMDSearch Admin · 当前用户{{ user?.username || '-' }}
</a-layout-footer>
</a-layout>
</a-layout>
</template>
<style scoped>
.main-layout {
min-height: 100vh;
}
.main-layout__sider {
position: sticky;
top: 0;
height: 100vh;
overflow: auto;
box-shadow: 2px 0 8px rgba(0, 21, 41, 0.15);
}
.main-layout__logo {
height: 52px;
display: flex;
align-items: center;
justify-content: center;
gap: 10px;
padding: 0 16px;
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
background: linear-gradient(135deg, #001529 0%, #002140 100%);
}
.main-layout__logo-icon {
display: inline-flex;
align-items: center;
justify-content: center;
width: 28px;
height: 28px;
border-radius: 6px;
background: linear-gradient(135deg, #1677ff 0%, #4096ff 100%);
color: #fff;
font-size: 16px;
box-shadow: 0 2px 6px rgba(22, 119, 255, 0.4);
}
.main-layout__logo-text {
color: #fff;
font-size: 17px;
font-weight: 700;
letter-spacing: 0.5px;
white-space: nowrap;
}
.main-layout__header {
display: flex;
align-items: center;
background: #fff;
padding: 0 20px;
height: 52px;
line-height: 52px;
box-shadow: 0 1px 4px rgba(0, 21, 41, 0.08);
z-index: 10;
}
.main-layout__collapse {
flex: 0 0 auto;
font-size: 16px;
color: #4b5563;
}
.main-layout__collapse:hover {
color: #1677ff;
}
.main-layout__title {
flex: 1 1 auto;
margin-left: 12px;
font-size: 16px;
font-weight: 600;
color: #1f2937;
}
.main-layout__user {
flex: 0 0 auto;
display: flex;
align-items: center;
gap: 4px;
}
.main-layout__user-name {
color: #4b5563;
font-size: 13px;
}
.main-layout__logout {
color: #6b7280;
}
.main-layout__logout:hover {
color: #ff4d4f;
}
.main-layout__content {
margin: 16px;
padding: 0;
background: transparent;
overflow: auto;
}
.main-layout__footer {
text-align: center;
color: #6b7280;
font-size: 12px;
background: transparent;
padding: 12px 16px;
}
</style>
+30
View File
@@ -0,0 +1,30 @@
import { createApp } from 'vue'
import { createPinia } from 'pinia'
import Antd from 'ant-design-vue'
import 'ant-design-vue/dist/reset.css'
import App from './App.vue'
import router from './router'
import { setUnauthorizedHandler } from './api/client'
import { useAuthStore } from './stores/useAuthStore'
import './styles/main.css'
const app = createApp(App)
const pinia = createPinia()
app.use(pinia)
// 注册 401 处理:清 store + 跳 /login
const authStore = useAuthStore()
setUnauthorizedHandler(() => {
authStore.clearAuth()
if (router.currentRoute.value.name !== 'login') {
router.replace({
name: 'login',
query: { redirect: router.currentRoute.value.fullPath }
})
}
})
app.use(router)
app.use(Antd)
app.mount('#app')
+91
View File
@@ -0,0 +1,91 @@
import { createRouter, createWebHistory } from 'vue-router'
import { useAuthStore } from '@/stores/useAuthStore'
const routes = [
{
path: '/login',
name: 'login',
component: () => import('@/views/Login.vue'),
meta: { title: '登录', requiresAuth: false }
},
{
path: '/',
component: () => import('@/layouts/MainLayout.vue'),
redirect: '/overview',
meta: { requiresAuth: true },
children: [
{
path: 'overview',
name: 'overview',
component: () => import('@/views/Overview.vue'),
meta: { title: '概览', requiresAuth: true }
},
{
path: 'documents',
name: 'documents',
component: () => import('@/views/Documents.vue'),
meta: { title: '文档管理', requiresAuth: true }
},
{
path: 'ingest',
name: 'ingest',
component: () => import('@/views/Ingest.vue'),
meta: { title: '文档入库', requiresAuth: true }
},
{
path: 'search',
name: 'search',
component: () => import('@/views/Search.vue'),
meta: { title: '检索测试台', requiresAuth: true }
},
{
path: 'categories',
name: 'categories',
component: () => import('@/views/Categories.vue'),
meta: { title: '类目列表', requiresAuth: true }
},
{
path: 'settings',
name: 'settings',
component: () => import('@/views/Settings.vue'),
meta: { title: '设置', requiresAuth: true }
}
]
},
{
path: '/:pathMatch(.*)*',
name: 'not-found',
redirect: '/overview'
}
]
const router = createRouter({
history: createWebHistory('/admin/'),
routes,
scrollBehavior() {
return { top: 0 }
}
})
// 全局前置守卫:未登录跳 /login;已登录访问 /login 跳 /overview
router.beforeEach((to) => {
const authStore = useAuthStore()
const title = to.meta?.title
if (title) {
document.title = `${title} - QMDSearch 知识库后台`
} else {
document.title = 'QMDSearch 知识库后台'
}
if (to.meta?.requiresAuth && !authStore.isAuthenticated) {
return { name: 'login', query: { redirect: to.fullPath } }
}
if (to.name === 'login' && authStore.isAuthenticated) {
return { name: 'overview' }
}
return true
})
export default router
+66
View File
@@ -0,0 +1,66 @@
import { defineStore } from 'pinia'
const TOKEN_STORAGE_KEY = 'qmd_token'
const USER_STORAGE_KEY = 'qmd_user'
/**
* 从 localStorage 读取用户信息
* @returns {{username:string, role:string, created_at?:string} | null}
*/
function loadUserFromStorage() {
try {
const raw = localStorage.getItem(USER_STORAGE_KEY)
if (!raw) return null
const parsed = JSON.parse(raw)
if (parsed && typeof parsed === 'object' && parsed.username) {
return parsed
}
return null
} catch {
return null
}
}
export const useAuthStore = defineStore('auth', {
state: () => ({
token: localStorage.getItem(TOKEN_STORAGE_KEY) || '',
user: loadUserFromStorage()
}),
getters: {
isAuthenticated: (state) => Boolean(state.token),
isAdmin: (state) => state.user?.role === 'admin',
displayName: (state) => {
if (!state.user) return ''
return `${state.user.username} (${state.user.role})`
}
},
actions: {
/**
* 登录成功后保存 token + user
* @param {{access_token:string, user:object}} data
*/
setAuth(data) {
this.token = data.access_token
this.user = data.user
localStorage.setItem(TOKEN_STORAGE_KEY, data.access_token)
localStorage.setItem(USER_STORAGE_KEY, JSON.stringify(data.user))
},
/** 清除登录态(登出 / 401) */
clearAuth() {
this.token = ''
this.user = null
localStorage.removeItem(TOKEN_STORAGE_KEY)
localStorage.removeItem(USER_STORAGE_KEY)
},
/**
* 登出
*/
logout() {
this.clearAuth()
}
}
})
+79
View File
@@ -0,0 +1,79 @@
html,
body,
#app {
height: 100%;
margin: 0;
padding: 0;
}
#app {
font-family: -apple-system, BlinkMacSystemFont, 'PingFang SC', 'Microsoft YaHei',
'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
color: #1f2937;
background: #f0f2f5;
}
/* 统一文本工具类 */
.text-muted {
color: #6b7280;
font-size: 12px;
}
.text-break {
word-break: break-word;
white-space: pre-wrap;
}
/* 页面通用 section 容器 */
.page-section {
background: #fff;
border-radius: 8px;
padding: 20px 24px;
box-shadow: 0 1px 2px rgba(0, 21, 41, 0.04);
}
/* flex-gap:横排卡片/标签 */
.flex-gap {
display: flex;
flex-wrap: wrap;
gap: 12px;
}
.toolbar {
margin: 12px 0;
}
/* 统一页面标题样式 */
.page-title {
font-size: 18px;
font-weight: 600;
margin: 0;
color: #1f2937;
}
/* 统一卡片标题样式 */
.section-subtitle {
font-size: 14px;
font-weight: 600;
color: #374151;
margin: 16px 0 12px;
}
/* 滚动条美化 */
::-webkit-scrollbar {
width: 8px;
height: 8px;
}
::-webkit-scrollbar-thumb {
background: rgba(0, 0, 0, 0.2);
border-radius: 4px;
}
::-webkit-scrollbar-thumb:hover {
background: rgba(0, 0, 0, 0.35);
}
::-webkit-scrollbar-track {
background: transparent;
}
+38
View File
@@ -0,0 +1,38 @@
/**
* 截断文本,超过最大长度追加省略号
* @param {string} text
* @param {number} [maxLen=80]
* @returns {string}
*/
export function truncate(text, maxLen = 80) {
if (!text) return ''
const s = String(text)
return s.length > maxLen ? `${s.slice(0, maxLen)}` : s
}
/**
* 安全拼接类名
* @param {...(string | false | null | undefined)} args
* @returns {string}
*/
export function classnames(...args) {
return args.filter(Boolean).join(' ')
}
/**
* 防抖
* @param {Function} fn
* @param {number} [wait=300]
* @returns {Function}
*/
export function debounce(fn, wait = 300) {
let timer = null
return function debounced(...args) {
if (timer) {
clearTimeout(timer)
}
timer = setTimeout(() => {
fn.apply(this, args)
}, wait)
}
}
+70
View File
@@ -0,0 +1,70 @@
<script setup>
import { onMounted, ref } from 'vue'
import { message } from 'ant-design-vue'
import { categories as fetchCategories } from '@/api/knowledge'
const isLoading = ref(false)
const categoriesData = ref([])
const columns = [
{ title: '名称', dataIndex: 'name', key: 'name' },
{ title: '描述', dataIndex: 'description', key: 'description' }
]
async function loadCategories() {
isLoading.value = true
try {
const data = await fetchCategories()
categoriesData.value = data.categories || []
} catch (err) {
message.error(err?.message || '加载类目列表失败')
} finally {
isLoading.value = false
}
}
onMounted(() => {
loadCategories()
})
</script>
<template>
<div class="categories page-section">
<div class="categories__header">
<h2 class="categories__title">类目列表</h2>
<a-button :loading="isLoading" @click="loadCategories">刷新</a-button>
</div>
<a-table
:columns="columns"
:data-source="categoriesData"
:pagination="false"
:loading="isLoading"
row-key="name"
size="small"
>
<template #bodyCell="{ column, record }">
<template v-if="column.key === 'name'">
<a-tag color="blue">{{ record.name }}</a-tag>
</template>
<template v-else-if="column.key === 'description'">
<span>{{ record.description || '-' }}</span>
</template>
</template>
</a-table>
</div>
</template>
<style scoped>
.categories__header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 12px;
}
.categories__title {
font-size: 16px;
margin: 0;
}
</style>
+299
View File
@@ -0,0 +1,299 @@
<script setup>
import { onMounted, reactive, ref } from 'vue'
import { message, Modal } from 'ant-design-vue'
import {
list as fetchDocuments,
detail as fetchDocumentDetail,
remove as deleteDocument
} from '@/api/documents'
import { truncate } from '@/utils/format'
const isLoading = ref(false)
const isLoadingDetail = ref(false)
const isDeleting = ref(false)
const documents = ref([])
const nextOffset = ref(null)
const detailVisible = ref(false)
const detailData = ref(null)
const statusText = reactive({
text: ''
})
function updateStatusText() {
statusText.text =
nextOffset.value === null ? '已加载全部' : '还有更多,可继续加载'
}
async function loadDocuments(reset = false) {
isLoading.value = true
try {
const offset = reset ? null : nextOffset.value
const data = await fetchDocuments(20, offset)
if (reset) {
documents.value = data.items || []
} else {
documents.value = documents.value.concat(data.items || [])
}
nextOffset.value = data.next_offset ?? null
updateStatusText()
} catch (err) {
message.error(err?.message || '加载文档列表失败')
} finally {
isLoading.value = false
}
}
async function handleLoadMore() {
await loadDocuments(false)
}
function handleViewDetail(docId) {
detailVisible.value = true
detailData.value = null
loadDocDetail(docId)
}
async function loadDocDetail(docId) {
isLoadingDetail.value = true
try {
detailData.value = await fetchDocumentDetail(docId)
} catch (err) {
message.error(err?.message || '加载文档详情失败')
} finally {
isLoadingDetail.value = false
}
}
function handleDelete(doc) {
const title = doc.title || doc.doc_id
Modal.confirm({
title: '确认删除',
content: `确定删除文档「${title}」(${doc.doc_id}) 吗?该操作将删除四层集合中的全部数据,不可恢复。`,
okText: '删除',
okType: 'danger',
cancelText: '取消',
async onOk() {
isDeleting.value = true
try {
const data = await deleteDocument(doc.doc_id)
message.success(`已删除 ${data.deleted_total ?? 0} 条数据`)
detailVisible.value = false
await loadDocuments(true)
} catch (err) {
message.error(err?.message || '删除文档失败')
} finally {
isDeleting.value = false
}
}
})
}
function handleCloseDetail() {
detailVisible.value = false
detailData.value = null
}
const columns = [
{ title: '标题', dataIndex: 'title', key: 'title', ellipsis: true },
{ title: '类目', dataIndex: 'category', key: 'category', width: 140 },
{ title: '标签', dataIndex: 'tags', key: 'tags', width: 220 },
{ title: 'L1 摘要', dataIndex: 'summary', key: 'summary', ellipsis: true },
{ title: '操作', key: 'action', width: 160, fixed: 'right' }
]
function getDocTitle(record) {
return record?.title || '(无标题)'
}
onMounted(() => {
loadDocuments(true)
})
</script>
<template>
<div class="documents page-section">
<div class="documents__header">
<h2 class="documents__title">文档管理</h2>
</div>
<a-table
:columns="columns"
:data-source="documents"
:pagination="false"
:loading="isLoading"
row-key="doc_id"
size="small"
:scroll="{ x: 900 }"
>
<template #bodyCell="{ column, record }">
<template v-if="column.key === 'title'">
<span :title="record.title">{{ getDocTitle(record) }}</span>
</template>
<template v-else-if="column.key === 'category'">
<a-tag v-if="record.category" color="blue">{{ record.category }}</a-tag>
<span v-else class="text-muted">-</span>
</template>
<template v-else-if="column.key === 'tags'">
<template v-if="record.tags && record.tags.length">
<a-tag v-for="tag in record.tags" :key="tag">{{ tag }}</a-tag>
</template>
<span v-else class="text-muted">-</span>
</template>
<template v-else-if="column.key === 'summary'">
<span :title="record.summary">{{ truncate(record.summary, 80) }}</span>
</template>
<template v-else-if="column.key === 'action'">
<a-space>
<a-button type="link" size="small" @click="handleViewDetail(record.doc_id)">
详情
</a-button>
<a-button
type="link"
size="small"
danger
:loading="isDeleting"
@click="handleDelete(record)"
>
删除
</a-button>
</a-space>
</template>
</template>
</a-table>
<div class="toolbar documents__toolbar">
<a-button
:loading="isLoading"
:disabled="nextOffset === null"
@click="handleLoadMore"
>
加载更多
</a-button>
<span class="text-muted" style="margin-left: 12px">{{ statusText.text }}</span>
</div>
<a-drawer
:open="detailVisible"
title="文档详情"
placement="right"
width="640"
:destroy-on-close="true"
@close="handleCloseDetail"
>
<a-spin :spinning="isLoadingDetail">
<div v-if="detailData">
<a-descriptions :column="1" size="small" bordered>
<a-descriptions-item label="doc_id">
{{ detailData.l1?.doc_id || '-' }}
</a-descriptions-item>
<a-descriptions-item label="标题">
{{ detailData.l1?.title || '-' }}
</a-descriptions-item>
<a-descriptions-item label="类目">
{{ detailData.l1?.category || '-' }}
</a-descriptions-item>
<a-descriptions-item label="标签">
<template v-if="detailData.l1?.tags && detailData.l1.tags.length">
<a-tag v-for="tag in detailData.l1.tags" :key="tag">{{ tag }}</a-tag>
</template>
<span v-else class="text-muted">-</span>
</a-descriptions-item>
<a-descriptions-item label="chunks_count">
{{ detailData.chunks_count ?? 0 }}
</a-descriptions-item>
</a-descriptions>
<h3 class="documents__section-title">L1 全文</h3>
<pre class="documents__pre">{{ detailData.l1?.text || '' }}</pre>
<h3 class="documents__section-title">
L2 节点{{ (detailData.l2_nodes || []).length }}
</h3>
<div v-if="(detailData.l2_nodes || []).length === 0" class="text-muted"></div>
<div
v-for="(node, idx) in detailData.l2_nodes || []"
:key="`l2-${idx}`"
class="documents__node"
>
<div class="documents__node-path">{{ node.section_path || '' }}</div>
<div class="documents__node-text text-break">{{ node.text || '' }}</div>
</div>
<h3 class="documents__section-title">
L3 节点{{ (detailData.l3_nodes || []).length }}
</h3>
<div v-if="(detailData.l3_nodes || []).length === 0" class="text-muted"></div>
<div
v-for="(node, idx) in detailData.l3_nodes || []"
:key="`l3-${idx}`"
class="documents__node"
>
<div class="documents__node-path">{{ node.section_path || '' }}</div>
<div class="documents__node-text text-break">{{ node.text || '' }}</div>
</div>
</div>
<div v-else-if="!isLoadingDetail" class="text-muted">暂无数据</div>
</a-spin>
</a-drawer>
</div>
</template>
<style scoped>
.documents__header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 12px;
}
.documents__title {
font-size: 16px;
margin: 0;
}
.documents__toolbar {
display: flex;
align-items: center;
}
.documents__section-title {
font-size: 14px;
margin: 16px 0 8px;
color: #374151;
}
.documents__pre {
background: #f9fafb;
border: 1px solid #e5e7eb;
border-radius: 4px;
padding: 8px;
white-space: pre-wrap;
word-break: break-word;
font-size: 12px;
max-height: 260px;
overflow: auto;
margin: 0;
}
.documents__node {
border-left: 3px solid #93c5fd;
padding: 6px 10px;
margin-bottom: 6px;
background: #f8fafc;
border-radius: 0 4px 4px 0;
}
.documents__node-path {
font-size: 12px;
color: #6b7280;
margin-bottom: 4px;
}
.documents__node-text {
font-size: 13px;
color: #1f2937;
}
</style>
+352
View File
@@ -0,0 +1,352 @@
<script setup>
import { computed, reactive, ref } from 'vue'
import { message } from 'ant-design-vue'
import { ingest as ingestDocument, upload as uploadDocument } from '@/api/documents'
import { useIngestPolling } from '@/composables/useIngestPolling'
import {
INGEST_STATUS_TEXT,
INGEST_STATUS_COLOR
} from '@/constants/ingest'
const { state: pollState, startPolling, stopPolling } = useIngestPolling()
const activeTab = ref('text')
const isSubmittingText = ref(false)
const isSubmittingFile = ref(false)
const textFormRef = ref(null)
const textForm = reactive({
title: '',
source: '',
text: ''
})
const textRules = {
title: [{ required: true, message: '请输入标题', trigger: 'blur' }],
text: [{ required: true, message: '请输入正文', trigger: 'blur' }]
}
const fileForm = reactive({
title: '',
source: ''
})
const fileList = ref([])
const rawFile = ref(null)
const ACCEPTED_EXTENSIONS = '.txt,.md,.html,.htm,.pdf,.docx'
const statusBadgeColor = computed(() => {
return INGEST_STATUS_COLOR[pollState.status] || 'default'
})
const statusBadgeText = computed(() => {
return INGEST_STATUS_TEXT[pollState.status] || pollState.status || '-'
})
const isTerminal = computed(() =>
['done', 'failed'].includes(pollState.status)
)
const resultData = computed(() => pollState.task?.result || null)
const errorData = computed(() => pollState.task?.error || pollState.error || null)
const summaryData = computed(() => resultData.value?.summary || {})
const partialSummary = computed(() => errorData.value?.partial_summary || null)
function handleFileChange(file) {
// a-upload before-upload 返回 false 表示不自动上传;保存原始 File 供 FormData 使用
rawFile.value = file
fileList.value = [file]
return false
}
function handleFileRemove() {
rawFile.value = null
fileList.value = []
}
async function handleSubmitText() {
try {
await textFormRef.value.validate()
} catch {
return
}
isSubmittingText.value = true
try {
const payload = {
title: textForm.title,
text: textForm.text
}
if (textForm.source) {
payload.source = textForm.source
}
const data = await ingestDocument(payload)
message.success('任务已提交')
startPolling(data.task_id)
} catch (err) {
message.error(err?.message || '提交入库失败')
} finally {
isSubmittingText.value = false
}
}
async function handleSubmitFile() {
if (!rawFile.value) {
message.warning('请选择文件')
return
}
isSubmittingFile.value = true
try {
const formData = new FormData()
formData.append('file', rawFile.value)
if (fileForm.title) {
formData.append('title', fileForm.title)
}
if (fileForm.source) {
formData.append('source', fileForm.source)
}
const data = await uploadDocument(formData)
message.success('任务已提交')
startPolling(data.task_id)
} catch (err) {
message.error(err?.message || '上传入库失败')
} finally {
isSubmittingFile.value = false
}
}
function handleCancelPolling() {
stopPolling()
message.info('已停止轮询')
}
</script>
<template>
<div class="ingest page-section">
<div class="ingest__header">
<h2 class="ingest__title">文档入库</h2>
</div>
<a-tabs v-model:activeKey="activeTab">
<a-tab-pane key="text" tab="文本入库">
<a-form
ref="textFormRef"
:model="textForm"
:rules="textRules"
layout="vertical"
>
<a-form-item label="标题" name="title">
<a-input v-model:value="textForm.title" placeholder="请输入标题" allow-clear />
</a-form-item>
<a-form-item label="来源" name="source">
<a-input
v-model:value="textForm.source"
placeholder="例如:manual / web / file"
allow-clear
/>
</a-form-item>
<a-form-item label="正文" name="text">
<a-textarea
v-model:value="textForm.text"
placeholder="请输入文档正文"
:auto-size="{ minRows: 8, maxRows: 18 }"
/>
</a-form-item>
<a-form-item>
<a-button
type="primary"
:loading="isSubmittingText || pollState.isPolling"
@click="handleSubmitText"
>
提交入库
</a-button>
</a-form-item>
</a-form>
</a-tab-pane>
<a-tab-pane key="file" tab="文件上传">
<a-form
:model="fileForm"
layout="vertical"
>
<a-form-item label="选择文件">
<a-upload
:file-list="fileList"
:accept="ACCEPTED_EXTENSIONS"
:max-count="1"
:before-upload="handleFileChange"
@remove="handleFileRemove"
>
<a-button :disabled="fileList.length >= 1">选择文件</a-button>
</a-upload>
<div class="text-muted" style="margin-top: 4px">
支持 .txt/.md/.html/.htm/.pdf/.docx
</div>
</a-form-item>
<a-form-item label="标题(可选,默认取文件名)" name="title">
<a-input
v-model:value="fileForm.title"
placeholder="留空则使用文件名去扩展"
allow-clear
/>
</a-form-item>
<a-form-item label="来源(可选,默认 file:原文件名)" name="source">
<a-input
v-model:value="fileForm.source"
placeholder="例如:manual / web"
allow-clear
/>
</a-form-item>
<a-form-item>
<a-button
type="primary"
:loading="isSubmittingFile || pollState.isPolling"
@click="handleSubmitFile"
>
上传入库
</a-button>
</a-form-item>
</a-form>
</a-tab-pane>
</a-tabs>
<div v-if="pollState.taskId" class="ingest__result">
<div class="ingest__result-header">
<span>任务已提交{{ pollState.taskId }}</span>
<a-tag :color="statusBadgeColor">{{ statusBadgeText }}</a-tag>
<a-button
v-if="pollState.isPolling"
type="link"
size="small"
@click="handleCancelPolling"
>
停止轮询
</a-button>
</div>
<a-alert
v-if="pollState.isTimeout"
class="ingest__alert"
type="warning"
show-icon
:message="`任务仍在进行,可稍后凭 task_id 查询:${pollState.taskId}`"
/>
<a-alert
v-if="pollState.error && !pollState.task"
class="ingest__alert"
type="error"
show-icon
:message="pollState.error?.message || '轮询失败'"
/>
<div v-if="isTerminal && pollState.status === 'done' && resultData" class="ingest__done">
<a-descriptions :column="1" size="small" bordered>
<a-descriptions-item label="document_id">
{{ resultData.document_id || '-' }}
</a-descriptions-item>
<a-descriptions-item label="类目">
{{ resultData.category || '-' }}置信度 {{ resultData.category_confidence ?? '-' }}
</a-descriptions-item>
<a-descriptions-item label="标签">
<template v-if="resultData.tags && resultData.tags.length">
<a-tag v-for="tag in resultData.tags" :key="tag">{{ tag }}</a-tag>
</template>
<span v-else class="text-muted">-</span>
</a-descriptions-item>
<a-descriptions-item label="总结层级">
{{ summaryData.level ?? '-' }}
</a-descriptions-item>
<a-descriptions-item label="写入集合">
{{ resultData.collection || '-' }}
</a-descriptions-item>
<a-descriptions-item label="chunks_count">
{{ resultData.chunks_count ?? 0 }}
</a-descriptions-item>
</a-descriptions>
<h3 class="ingest__section-title">L1 总结</h3>
<pre class="ingest__pre">{{ summaryData.l1_summary || '' }}</pre>
</div>
<div v-if="isTerminal && pollState.status === 'failed'" class="ingest__failed">
<a-descriptions :column="1" size="small" bordered>
<a-descriptions-item label="失败阶段">
{{ errorData?.stage || '-' }}
</a-descriptions-item>
<a-descriptions-item label="错误信息">
<span class="text-break">{{ errorData?.message || '-' }}</span>
</a-descriptions-item>
</a-descriptions>
<template v-if="partialSummary && partialSummary.l1_summary">
<a-alert
class="ingest__alert"
type="info"
show-icon
message="已产出总结保留:任务失败前已生成 L1 摘要"
/>
<h3 class="ingest__section-title">L1 总结</h3>
<pre class="ingest__pre">{{ partialSummary.l1_summary }}</pre>
</template>
</div>
</div>
</div>
</template>
<style scoped>
.ingest__header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 12px;
}
.ingest__title {
font-size: 16px;
margin: 0;
}
.ingest__result {
margin-top: 16px;
border: 1px solid #e5e7eb;
border-radius: 6px;
background: #fafafa;
padding: 12px 16px;
}
.ingest__result-header {
display: flex;
align-items: center;
gap: 8px;
flex-wrap: wrap;
font-size: 13px;
margin-bottom: 8px;
}
.ingest__alert {
margin: 8px 0;
}
.ingest__done,
.ingest__failed {
margin-top: 8px;
}
.ingest__section-title {
font-size: 14px;
margin: 12px 0 8px;
color: #374151;
}
.ingest__pre {
background: #f9fafb;
border: 1px solid #e5e7eb;
border-radius: 4px;
padding: 8px;
white-space: pre-wrap;
word-break: break-word;
font-size: 12px;
max-height: 260px;
overflow: auto;
margin: 0;
}
</style>
+199
View File
@@ -0,0 +1,199 @@
<script setup>
import { reactive, ref } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import { message } from 'ant-design-vue'
import { UserOutlined, LockOutlined, DatabaseOutlined } from '@ant-design/icons-vue'
import { login as loginApi } from '@/api/auth'
import { useAuthStore } from '@/stores/useAuthStore'
const route = useRoute()
const router = useRouter()
const authStore = useAuthStore()
const isLoading = ref(false)
const formRef = ref(null)
const formState = reactive({
username: '',
password: ''
})
const rules = {
username: [{ required: true, message: '请输入用户名', trigger: 'blur' }],
password: [{ required: true, message: '请输入密码', trigger: 'blur' }]
}
async function handleSubmit() {
try {
await formRef.value.validate()
} catch {
return
}
isLoading.value = true
try {
const data = await loginApi(formState.username, formState.password)
authStore.setAuth(data)
message.success('登录成功')
const redirect = typeof route.query.redirect === 'string' ? route.query.redirect : '/overview'
router.replace(redirect)
} catch (err) {
message.error(err?.message || '登录失败')
} finally {
isLoading.value = false
}
}
</script>
<template>
<div class="login-page">
<div class="login-page__bg-deco login-page__bg-deco--1" />
<div class="login-page__bg-deco login-page__bg-deco--2" />
<div class="login-page__box">
<div class="login-page__brand">
<div class="login-page__logo">
<DatabaseOutlined />
</div>
<div class="login-page__brand-text">
<div class="login-page__brand-name">QMDSearch</div>
<div class="login-page__brand-sub">知识库管理后台</div>
</div>
</div>
<a-form
ref="formRef"
:model="formState"
:rules="rules"
layout="vertical"
@finish="handleSubmit"
>
<a-form-item label="用户名" name="username">
<a-input
v-model:value="formState.username"
placeholder="请输入用户名"
autocomplete="username"
allow-clear
size="large"
>
<template #prefix><UserOutlined /></template>
</a-input>
</a-form-item>
<a-form-item label="密码" name="password">
<a-input-password
v-model:value="formState.password"
placeholder="请输入密码"
autocomplete="current-password"
size="large"
@pressEnter="handleSubmit"
>
<template #prefix><LockOutlined /></template>
</a-input-password>
</a-form-item>
<a-form-item>
<a-button
type="primary"
html-type="submit"
block
size="large"
:loading="isLoading"
>
登录
</a-button>
</a-form-item>
</a-form>
<div class="login-page__footer">
QMDSearch Admin · 分层信息检索服务
</div>
</div>
</div>
</template>
<style scoped>
.login-page {
position: relative;
min-height: 100vh;
display: flex;
align-items: center;
justify-content: center;
overflow: hidden;
background: linear-gradient(135deg, #1e3a8a 0%, #1e40af 35%, #0c4a6e 100%);
}
.login-page__bg-deco {
position: absolute;
border-radius: 50%;
filter: blur(80px);
opacity: 0.4;
pointer-events: none;
}
.login-page__bg-deco--1 {
width: 360px;
height: 360px;
background: #4096ff;
top: -120px;
right: -80px;
}
.login-page__bg-deco--2 {
width: 320px;
height: 320px;
background: #13c2c2;
bottom: -100px;
left: -60px;
}
.login-page__box {
position: relative;
z-index: 1;
width: 380px;
background: #fff;
border-radius: 12px;
padding: 32px 36px 24px;
box-shadow: 0 16px 48px rgba(0, 0, 0, 0.25);
}
.login-page__brand {
display: flex;
align-items: center;
gap: 14px;
margin-bottom: 28px;
}
.login-page__logo {
display: inline-flex;
align-items: center;
justify-content: center;
width: 48px;
height: 48px;
border-radius: 10px;
background: linear-gradient(135deg, #1677ff 0%, #4096ff 100%);
color: #fff;
font-size: 26px;
box-shadow: 0 6px 16px rgba(22, 119, 255, 0.4);
flex: 0 0 auto;
}
.login-page__brand-text {
display: flex;
flex-direction: column;
gap: 2px;
}
.login-page__brand-name {
font-size: 22px;
font-weight: 700;
color: #1f2937;
line-height: 1.1;
}
.login-page__brand-sub {
font-size: 13px;
color: #6b7280;
}
.login-page__footer {
text-align: center;
font-size: 12px;
color: #9ca3af;
margin-top: 8px;
}
</style>
+349
View File
@@ -0,0 +1,349 @@
<script setup>
import { computed, onMounted, ref } from 'vue'
import { message } from 'ant-design-vue'
import {
DatabaseOutlined,
ApartmentOutlined,
FileSearchOutlined,
BlockOutlined,
FileTextOutlined,
QuestionCircleOutlined,
ReloadOutlined,
AppstoreOutlined
} from '@ant-design/icons-vue'
import { stats as fetchStats } from '@/api/knowledge'
const isLoading = ref(false)
const statsData = ref(null)
const cardMeta = [
{
key: 'doc_l1',
label: 'L1 文档总结',
icon: DatabaseOutlined,
color: '#1677ff',
bg: 'rgba(22, 119, 255, 0.12)'
},
{
key: 'doc_l2',
label: 'L2 大纲节点',
icon: ApartmentOutlined,
color: '#722ed1',
bg: 'rgba(114, 46, 209, 0.12)'
},
{
key: 'doc_l3',
label: 'L3 内容大纲',
icon: FileSearchOutlined,
color: '#13c2c2',
bg: 'rgba(19, 194, 194, 0.12)'
},
{
key: 'chunks',
label: 'Chunks',
icon: BlockOutlined,
color: '#fa8c16',
bg: 'rgba(250, 140, 22, 0.12)'
},
{
key: '__documents_total',
label: '文档总数',
icon: FileTextOutlined,
color: '#52c41a',
bg: 'rgba(82, 196, 26, 0.12)'
},
{
key: '__uncategorized',
label: '未分类文档',
icon: QuestionCircleOutlined,
color: '#ff4d4f',
bg: 'rgba(255, 77, 79, 0.12)'
}
]
const cards = computed(() => {
const collections = statsData.value?.collections || {}
return cardMeta.map((m) => {
let value = 0
if (m.key === '__documents_total') {
value = statsData.value?.documents_total ?? 0
} else if (m.key === '__uncategorized') {
value = statsData.value?.uncategorized_count ?? 0
} else {
value = collections[m.key] ?? 0
}
return { ...m, value }
})
})
const categoryBars = computed(() => {
const categories = statsData.value?.categories || {}
const entries = Object.entries(categories).map(([name, count]) => ({ name, count }))
entries.sort((a, b) => b.count - a.count)
const max = entries.reduce((acc, item) => Math.max(acc, item.count), 1)
const total = entries.reduce((acc, item) => acc + item.count, 0)
return entries.map((item) => ({
...item,
percent: Math.round((item.count / max) * 100),
ratio: total ? Math.round((item.count / total) * 100) : 0
}))
})
const gradientColors = [
'linear-gradient(90deg, #1677ff 0%, #4096ff 100%)',
'linear-gradient(90deg, #722ed1 0%, #9254de 100%)',
'linear-gradient(90deg, #13c2c2 0%, #36cfc9 100%)',
'linear-gradient(90deg, #fa8c16 0%, #ffa940 100%)',
'linear-gradient(90deg, #52c41a 0%, #73d13d 100%)',
'linear-gradient(90deg, #eb2f96 0%, #f759ab 100%)',
'linear-gradient(90deg, #fa541c 0%, #ff7a45 100%)',
'linear-gradient(90deg, #2f54eb 0%, #597ef7 100%)',
'linear-gradient(90deg, #08979c 0%, #13c2c2 100%)',
'linear-gradient(90deg, #c41d7f 0%, #eb2f96 100%)'
]
function barGradient(index) {
return gradientColors[index % gradientColors.length]
}
async function loadStats() {
isLoading.value = true
try {
statsData.value = await fetchStats()
} catch (err) {
message.error(err?.message || '获取统计失败')
} finally {
isLoading.value = false
}
}
onMounted(() => {
loadStats()
})
</script>
<template>
<div class="overview page-section">
<div class="overview__header">
<h2 class="page-title">概览</h2>
<a-space>
<a-button :loading="isLoading" @click="loadStats">
<template #icon><ReloadOutlined /></template>
刷新
</a-button>
</a-space>
</div>
<a-spin :spinning="isLoading">
<div class="overview__cards">
<div
v-for="card in cards"
:key="card.key"
class="overview__card"
>
<div class="overview__card-body">
<div
class="overview__card-icon"
:style="{ background: card.bg, color: card.color }"
>
<component :is="card.icon" />
</div>
<div class="overview__card-info">
<div class="overview__card-num">{{ card.value }}</div>
<div class="overview__card-label">{{ card.label }}</div>
</div>
</div>
</div>
</div>
<div class="overview__section">
<div class="overview__section-head">
<h3 class="overview__subtitle">
<AppstoreOutlined />
<span>类目分布</span>
</h3>
<span v-if="categoryBars.length" class="text-muted overview__section-meta">
{{ categoryBars.length }} 个类目
</span>
</div>
<div v-if="categoryBars.length === 0" class="overview__empty text-muted">
暂无数据
</div>
<div v-else class="overview__bars">
<div
v-for="(bar, idx) in categoryBars"
:key="bar.name"
class="overview__bar-row"
:title="`${bar.name}: ${bar.count} (${bar.ratio}%)`"
>
<span class="overview__bar-name" :title="bar.name">{{ bar.name }}</span>
<div class="overview__bar-track">
<div
class="overview__bar-fill"
:style="{ width: bar.percent + '%', background: barGradient(idx) }"
/>
</div>
<span class="overview__bar-count">{{ bar.count }}</span>
<span class="overview__bar-ratio">{{ bar.ratio }}%</span>
</div>
</div>
</div>
</a-spin>
</div>
</template>
<style scoped>
.overview__header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 16px;
}
.overview__cards {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
gap: 16px;
margin-bottom: 24px;
}
.overview__card {
background: #fff;
border: 1px solid #f0f0f0;
border-radius: 8px;
padding: 16px 18px;
transition: all 0.2s;
box-shadow: 0 1px 2px rgba(0, 21, 41, 0.04);
}
.overview__card:hover {
box-shadow: 0 4px 12px rgba(0, 21, 41, 0.08);
transform: translateY(-2px);
}
.overview__card-body {
display: flex;
align-items: center;
gap: 14px;
}
.overview__card-icon {
display: inline-flex;
align-items: center;
justify-content: center;
width: 44px;
height: 44px;
border-radius: 8px;
font-size: 22px;
flex: 0 0 auto;
}
.overview__card-info {
display: flex;
flex-direction: column;
gap: 2px;
min-width: 0;
}
.overview__card-num {
font-size: 26px;
font-weight: 700;
color: #1f2937;
line-height: 1.1;
}
.overview__card-label {
font-size: 12px;
color: #6b7280;
}
.overview__section {
margin-top: 8px;
}
.overview__section-head {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 12px;
}
.overview__subtitle {
font-size: 14px;
font-weight: 600;
color: #374151;
margin: 0;
display: flex;
align-items: center;
gap: 6px;
}
.overview__section-meta {
font-size: 12px;
}
.overview__empty {
text-align: center;
padding: 32px 0;
}
.overview__bars {
display: flex;
flex-direction: column;
gap: 8px;
}
.overview__bar-row {
display: flex;
align-items: center;
font-size: 13px;
cursor: default;
}
.overview__bar-name {
width: 180px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
color: #374151;
flex: 0 0 auto;
}
.overview__bar-track {
flex: 1;
background: #f3f4f6;
border-radius: 4px;
height: 18px;
margin: 0 10px;
overflow: hidden;
}
.overview__bar-fill {
height: 100%;
border-radius: 4px;
min-width: 2px;
transition: width 0.4s ease;
}
.overview__bar-count {
width: 48px;
text-align: right;
color: #1f2937;
font-weight: 600;
flex: 0 0 auto;
}
.overview__bar-ratio {
width: 48px;
text-align: right;
color: #6b7280;
font-size: 12px;
flex: 0 0 auto;
}
@media (max-width: 768px) {
.overview__bar-name {
width: 120px;
}
}
</style>
+234
View File
@@ -0,0 +1,234 @@
<script setup>
import { reactive, ref } from 'vue'
import { message } from 'ant-design-vue'
import { search as searchApi } from '@/api/search'
const isLoading = ref(false)
const resultData = ref(null)
const formRef = ref(null)
const formState = reactive({
query: '',
top_k: 5,
summarize: false
})
const rules = {
query: [{ required: true, message: '请输入查询语句', trigger: 'blur' }],
top_k: [{ type: 'number', min: 1, max: 50, message: 'top_k 范围 1~50', trigger: 'change' }]
}
const routedCategoriesText = (cats) => {
if (!cats || !cats.length) return '(无)'
return cats.join(', ')
}
async function handleSearch() {
try {
await formRef.value.validate()
} catch {
return
}
isLoading.value = true
resultData.value = null
try {
const payload = {
query: formState.query,
top_k: formState.top_k,
summarize: formState.summarize
}
resultData.value = await searchApi(payload)
} catch (err) {
message.error(err?.message || '检索失败')
} finally {
isLoading.value = false
}
}
function eiValue(value) {
if (value === undefined || value === null || value === '') return '(无)'
if (Array.isArray(value)) {
return value.length ? value.join(', ') : '(无)'
}
return String(value)
}
const hits = (data) => data?.hits || []
</script>
<template>
<div class="search page-section">
<div class="search__header">
<h2 class="search__title">检索测试台</h2>
</div>
<a-form
ref="formRef"
:model="formState"
:rules="rules"
layout="vertical"
>
<a-form-item label="查询语句" name="query">
<a-input
v-model:value="formState.query"
placeholder="请输入查询语句"
allow-clear
@pressEnter="handleSearch"
/>
</a-form-item>
<a-form-item label="top_k" name="top_k">
<a-input-number
v-model:value="formState.top_k"
:min="1"
:max="50"
style="width: 160px"
/>
</a-form-item>
<a-form-item name="summarize">
<a-checkbox v-model:checked="formState.summarize">
对结果生成 AI 总结
</a-checkbox>
</a-form-item>
<a-form-item>
<a-button type="primary" :loading="isLoading" @click="handleSearch">
检索
</a-button>
</a-form-item>
</a-form>
<div v-if="resultData" class="search__result">
<a-descriptions :column="1" size="small" bordered>
<a-descriptions-item label="routed_categories">
{{ routedCategoriesText(resultData.routed_categories) }}
</a-descriptions-item>
</a-descriptions>
<a-alert
v-if="resultData.fallback"
class="search__alert"
type="warning"
show-icon
message="fallback:本次检索触发了回退策略(路由类目无命中,已降级全局检索)"
/>
<div v-if="resultData.summary" class="search__summary">
<div class="search__summary-title">AI 总结</div>
<div class="text-break">{{ resultData.summary }}</div>
</div>
<div v-if="resultData.extracted_info" class="search__extracted">
<div class="search__extracted-title">AI 提取的关键信息</div>
<a-descriptions :column="1" size="small" bordered>
<a-descriptions-item label="改写">
{{ eiValue(resultData.extracted_info.rewrite) }}
</a-descriptions-item>
<a-descriptions-item label="关键词">
{{ eiValue(resultData.extracted_info.keywords) }}
</a-descriptions-item>
<a-descriptions-item label="实体">
{{ eiValue(resultData.extracted_info.entities) }}
</a-descriptions-item>
<a-descriptions-item label="意图">
{{ eiValue(resultData.extracted_info.intent) }}
</a-descriptions-item>
<a-descriptions-item label="时间范围">
{{ eiValue(resultData.extracted_info.time_range) }}
</a-descriptions-item>
<a-descriptions-item label="命中类目">
{{ eiValue(resultData.extracted_info.categories) }}
</a-descriptions-item>
</a-descriptions>
</div>
<div class="text-muted" style="margin: 12px 0 8px">
命中 {{ hits(resultData).length }}
</div>
<div
v-for="(hit, idx) in hits(resultData)"
:key="`hit-${idx}`"
class="search__hit"
>
<div class="search__hit-meta">
score={{ hit.score }} | doc_id={{ hit.doc_id }} | 标题={{ hit.title || '' }} | section={{ hit.section_path || '' }}
</div>
<div class="search__hit-snippet text-break">{{ hit.text || '' }}</div>
<div v-if="hit.doc_summary" class="search__hit-meta">
文档摘要{{ hit.doc_summary }}
</div>
</div>
</div>
</div>
</template>
<style scoped>
.search__header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 12px;
}
.search__title {
font-size: 16px;
margin: 0;
}
.search__result {
margin-top: 16px;
}
.search__alert {
margin: 12px 0;
}
.search__summary {
background: #f0fdf4;
border: 1px solid #bbf7d0;
border-radius: 6px;
padding: 12px 14px;
margin: 12px 0;
font-size: 13px;
}
.search__summary-title {
font-weight: 600;
margin-bottom: 6px;
color: #15803d;
}
.search__extracted {
background: #f0f9ff;
border: 1px solid #bae6fd;
border-radius: 6px;
padding: 12px 14px;
margin: 12px 0;
font-size: 13px;
}
.search__extracted-title {
font-weight: 600;
margin-bottom: 8px;
color: #0369a1;
}
.search__hit {
border: 1px solid #e5e7eb;
border-radius: 6px;
padding: 10px 12px;
margin-bottom: 10px;
background: #fff;
}
.search__hit-meta {
font-size: 12px;
color: #6b7280;
margin-bottom: 6px;
word-break: break-all;
}
.search__hit-snippet {
font-size: 13px;
color: #1f2937;
}
</style>
+489
View File
@@ -0,0 +1,489 @@
<script setup>
import { computed, onMounted, reactive, ref } from 'vue'
import { message, Modal } from 'ant-design-vue'
import { storeToRefs } from 'pinia'
import {
RobotOutlined,
FileTextOutlined,
FilterOutlined,
ReloadOutlined,
SaveOutlined,
UndoOutlined,
LockOutlined
} from '@ant-design/icons-vue'
import {
get as getSettings,
update as updateSettings,
schema as getSettingsSchema,
reset as resetSettings
} from '@/api/settings'
import { useAuthStore } from '@/stores/useAuthStore'
const authStore = useAuthStore()
const { isAdmin } = storeToRefs(authStore)
const activeTab = ref('models')
const isLoading = ref(false)
const isSaving = ref(false)
const isResetting = ref(false)
const schemaData = ref(null)
const DEFAULT_MODEL_CONFIG = {
provider: 'ollama',
base_url: '',
api_key: '',
model: '',
timeout: 120,
temperature: 0.3
}
const form = reactive({
models: {
summarize: { ...DEFAULT_MODEL_CONFIG },
query: { ...DEFAULT_MODEL_CONFIG },
classify: { ...DEFAULT_MODEL_CONFIG }
},
parsers: {
ocr: { plugin: '', params: {} },
pdf: { plugin: '', params: {} },
docx: { plugin: '', params: {} }
},
dedup: {
strategy: 'sha256',
simhash_threshold: 3,
ttl_seconds: 86400
}
})
const modelKeys = ['summarize', 'query', 'classify']
const modelMeta = {
summarize: {
label: '总结模型',
key: 'summarize',
desc: '用于文档三级总结',
icon: RobotOutlined,
color: '#1677ff'
},
query: {
label: '查询模型',
key: 'query',
desc: '用于 query 解析与重写',
icon: FilterOutlined,
color: '#722ed1'
},
classify: {
label: '分类模型',
key: 'classify',
desc: '用于文档分类判定',
icon: FileTextOutlined,
color: '#13c2c2'
}
}
const llmProviderOptions = computed(() => {
return (schemaData.value?.llm_providers || ['ollama', 'openai_compatible']).map(
(p) => ({ label: p, value: p })
)
})
const ocrPluginOptions = computed(() =>
(schemaData.value?.ocr_plugins || []).map((p) => ({ label: p, value: p }))
)
const pdfPluginOptions = computed(() =>
(schemaData.value?.pdf_plugins || []).map((p) => ({ label: p, value: p }))
)
const docxPluginOptions = computed(() =>
(schemaData.value?.docx_plugins || []).map((p) => ({ label: p, value: p }))
)
const dedupStrategyOptions = computed(() =>
(schemaData.value?.dedup_strategies || ['none', 'sha256', 'simhash']).map((s) => ({
label: s,
value: s
}))
)
const isSimhash = computed(() => form.dedup.strategy === 'simhash')
function applySettingsToForm(cfg) {
if (!cfg || typeof cfg !== 'object') return
if (cfg.models && typeof cfg.models === 'object') {
for (const key of modelKeys) {
const src = cfg.models[key] || {}
form.models[key] = {
provider: src.provider ?? DEFAULT_MODEL_CONFIG.provider,
base_url: src.base_url ?? '',
api_key: src.api_key ?? '',
model: src.model ?? '',
timeout: src.timeout ?? 120,
temperature: src.temperature ?? 0.3
}
}
}
if (cfg.parsers && typeof cfg.parsers === 'object') {
const ocr = cfg.parsers.ocr || {}
const pdf = cfg.parsers.pdf || {}
const docx = cfg.parsers.docx || {}
form.parsers.ocr = { plugin: ocr.plugin ?? '', params: ocr.params || {} }
form.parsers.pdf = { plugin: pdf.plugin ?? '', params: pdf.params || {} }
form.parsers.docx = { plugin: docx.plugin ?? '', params: docx.params || {} }
}
if (cfg.dedup && typeof cfg.dedup === 'object') {
form.dedup.strategy = cfg.dedup.strategy ?? 'sha256'
form.dedup.simhash_threshold = cfg.dedup.simhash_threshold ?? 3
form.dedup.ttl_seconds = cfg.dedup.ttl_seconds ?? 86400
}
}
function buildPayload() {
return {
models: {
summarize: { ...form.models.summarize },
query: { ...form.models.query },
classify: { ...form.models.classify }
},
parsers: {
ocr: { plugin: form.parsers.ocr.plugin, params: { ...form.parsers.ocr.params } },
pdf: { plugin: form.parsers.pdf.plugin, params: { ...form.parsers.pdf.params } },
docx: { plugin: form.parsers.docx.plugin, params: { ...form.parsers.docx.params } }
},
dedup: {
strategy: form.dedup.strategy,
simhash_threshold: form.dedup.simhash_threshold,
ttl_seconds: form.dedup.ttl_seconds
}
}
}
async function loadAll() {
isLoading.value = true
try {
const [cfg, schema] = await Promise.all([getSettings(), getSettingsSchema()])
schemaData.value = schema
applySettingsToForm(cfg)
} catch (err) {
message.error(err?.message || '加载设置失败')
} finally {
isLoading.value = false
}
}
async function handleSave() {
if (!isAdmin.value) {
message.error('仅管理员可修改设置')
return
}
isSaving.value = true
try {
const payload = buildPayload()
const cfg = await updateSettings(payload)
applySettingsToForm(cfg)
message.success('设置已保存')
} catch (err) {
message.error(err?.message || '保存设置失败')
} finally {
isSaving.value = false
}
}
function handleReset() {
if (!isAdmin.value) {
message.error('仅管理员可重置设置')
return
}
Modal.confirm({
title: '确认重置',
content: '确定要将所有运行时设置重置为默认值吗?此操作不可撤销。',
okText: '重置',
okType: 'danger',
cancelText: '取消',
async onOk() {
isResetting.value = true
try {
const cfg = await resetSettings()
applySettingsToForm(cfg)
message.success('已重置为默认值')
} catch (err) {
message.error(err?.message || '重置失败')
} finally {
isResetting.value = false
}
}
})
}
onMounted(() => {
loadAll()
})
</script>
<template>
<div class="settings page-section">
<div class="settings__header">
<h2 class="page-title">设置</h2>
<a-space>
<a-button :loading="isLoading" @click="loadAll">
<template #icon><ReloadOutlined /></template>
刷新
</a-button>
<a-button
type="primary"
:loading="isSaving"
:disabled="!isAdmin"
@click="handleSave"
>
<template #icon><SaveOutlined /></template>
保存
</a-button>
<a-button
danger
:loading="isResetting"
:disabled="!isAdmin"
@click="handleReset"
>
<template #icon><UndoOutlined /></template>
重置默认
</a-button>
</a-space>
</div>
<a-alert
v-if="!isAdmin"
class="settings__alert"
type="info"
show-icon
message="当前用户非管理员,仅可查看设置;保存与重置需 admin 权限。"
/>
<a-spin :spinning="isLoading">
<a-tabs v-model:activeKey="activeTab">
<a-tab-pane key="models" tab="模型配置">
<div class="settings__cards">
<a-card
v-for="key in modelKeys"
:key="key"
class="settings__card"
:body-style="{ padding: '16px 18px' }"
>
<template #title>
<div class="settings__card-title">
<span
class="settings__card-icon"
:style="{ background: modelMeta[key].color }"
>
<component :is="modelMeta[key].icon" />
</span>
<div class="settings__card-head">
<div class="settings__card-name">{{ modelMeta[key].label }}</div>
<div class="settings__card-desc">{{ modelMeta[key].desc }}</div>
</div>
</div>
</template>
<a-form layout="vertical" :colon="false">
<a-form-item label="provider">
<a-radio-group v-model:value="form.models[key].provider" button-style="solid">
<a-radio
v-for="opt in llmProviderOptions"
:key="opt.value"
:value="opt.value"
>
{{ opt.label }}
</a-radio>
</a-radio-group>
</a-form-item>
<a-row :gutter="12">
<a-col :span="24">
<a-form-item label="base_url">
<a-input
v-model:value="form.models[key].base_url"
placeholder="例如 http://localhost:11434"
allow-clear
/>
</a-form-item>
</a-col>
</a-row>
<a-form-item
v-if="form.models[key].provider !== 'ollama'"
label="api_key"
>
<a-input-password
v-model:value="form.models[key].api_key"
placeholder="无则留空"
allow-clear
>
<template #prefix><LockOutlined /></template>
</a-input-password>
</a-form-item>
<a-row :gutter="12">
<a-col :span="14">
<a-form-item label="model">
<a-input
v-model:value="form.models[key].model"
placeholder="例如 qwen2.5:1.5b"
allow-clear
/>
</a-form-item>
</a-col>
<a-col :span="10">
<a-form-item label="timeout (秒)">
<a-input-number
v-model:value="form.models[key].timeout"
:min="1"
style="width: 100%"
/>
</a-form-item>
</a-col>
</a-row>
<a-form-item label="temperature">
<a-input-number
v-model:value="form.models[key].temperature"
:min="0"
:max="2"
:step="0.1"
style="width: 100%"
/>
</a-form-item>
</a-form>
</a-card>
</div>
</a-tab-pane>
<a-tab-pane key="parsers" tab="解析插件">
<a-form layout="vertical" class="settings__parsers">
<a-form-item label="OCR 插件">
<a-select
v-model:value="form.parsers.ocr.plugin"
:options="ocrPluginOptions"
placeholder="选择 OCR 插件"
allow-clear
/>
</a-form-item>
<a-form-item label="PDF 插件">
<a-select
v-model:value="form.parsers.pdf.plugin"
:options="pdfPluginOptions"
placeholder="选择 PDF 插件"
allow-clear
/>
</a-form-item>
<a-form-item label="DOCX 插件">
<a-select
v-model:value="form.parsers.docx.plugin"
:options="docxPluginOptions"
placeholder="选择 DOCX 插件"
allow-clear
/>
</a-form-item>
</a-form>
</a-tab-pane>
<a-tab-pane key="dedup" tab="去重策略">
<a-form layout="vertical" class="settings__dedup">
<a-form-item label="strategy">
<a-select
v-model:value="form.dedup.strategy"
:options="dedupStrategyOptions"
placeholder="选择去重策略"
/>
</a-form-item>
<a-form-item label="simhash_threshold (0~64)">
<a-input-number
v-model:value="form.dedup.simhash_threshold"
:min="0"
:max="64"
:disabled="!isSimhash"
style="width: 100%"
/>
<span v-if="!isSimhash" class="text-muted" style="margin-top: 4px; display: inline-block">
strategy=simhash 时启用
</span>
</a-form-item>
<a-form-item label="ttl_seconds">
<a-input-number
v-model:value="form.dedup.ttl_seconds"
:min="0"
style="width: 100%"
/>
</a-form-item>
</a-form>
</a-tab-pane>
</a-tabs>
</a-spin>
</div>
</template>
<style scoped>
.settings__header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 16px;
}
.settings__alert {
margin-bottom: 12px;
}
.settings__cards {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(320px, 1fr));
gap: 16px;
}
.settings__card {
border-radius: 8px;
border: 1px solid #e5e7eb;
transition: box-shadow 0.2s;
}
.settings__card:hover {
box-shadow: 0 4px 12px rgba(0, 21, 41, 0.06);
}
.settings__card :deep(.ant-card-head) {
padding: 12px 16px;
border-bottom: 1px solid #f0f0f0;
min-height: auto;
}
.settings__card-title {
display: flex;
align-items: center;
gap: 10px;
}
.settings__card-icon {
display: inline-flex;
align-items: center;
justify-content: center;
width: 30px;
height: 30px;
border-radius: 6px;
color: #fff;
font-size: 16px;
flex: 0 0 auto;
}
.settings__card-head {
display: flex;
flex-direction: column;
gap: 2px;
}
.settings__card-name {
font-size: 14px;
font-weight: 600;
color: #1f2937;
line-height: 1.2;
}
.settings__card-desc {
font-size: 12px;
color: #6b7280;
line-height: 1.2;
}
.settings__parsers,
.settings__dedup {
max-width: 480px;
}
</style>
+32
View File
@@ -0,0 +1,32 @@
import { fileURLToPath, URL } from 'node:url'
import { defineConfig } from 'vite'
import vue from '@vitejs/plugin-vue'
// Vite 配置:base 部署到 /admin/dev 下 /api 与 /admin 代理到后端 8000
export default defineConfig({
base: '/admin/',
plugins: [vue()],
resolve: {
alias: {
'@': fileURLToPath(new URL('./src', import.meta.url))
}
},
server: {
port: 5173,
proxy: {
'/api': {
target: 'http://localhost:8000',
changeOrigin: true
},
'/admin': {
target: 'http://localhost:8000',
changeOrigin: true
}
}
},
build: {
outDir: 'dist',
sourcemap: false,
chunkSizeWarningLimit: 1500
}
})
+24
View File
@@ -5,6 +5,9 @@ DELETE /documents 加了 Depends(require_admin)。这里通过 autouse 夹具把
统一替换为返回固定 admin AuthUser 的 lambda,使现有 API 测试无需改动即可通过认证。
单个测试需要走真实认证逻辑时(如 tests/test_auth.py),可在测试函数内
pop 掉对应 overrideautouse fixture yield 后会统一 clear。
另外清理 LLM 客户端 / 解析插件 / 去重策略三处进程级缓存,避免跨测试串扰
(不同测试可能注入不同的 redis/ollama 实例,缓存按配置签名而非实例区分)。
"""
from datetime import UTC, datetime
@@ -25,3 +28,24 @@ def override_auth():
app.dependency_overrides[require_admin] = lambda: TEST_USER
yield
app.dependency_overrides.clear()
@pytest.fixture(autouse=True)
def _invalidate_runtime_caches():
"""每个测试前后清理 LLM/解析插件/去重策略进程级缓存
这三处缓存按 runtime_settings 配置签名而非实例区分,跨测试若配置相同
会复用旧实例(绑定到上个测试的 redis/ollama 替身),导致串扰。
"""
# 延迟导入避免循环依赖
from app.core.dedup import invalidate_dedup_strategy_cache
from app.core.file_parser import invalidate_parser_plugin_cache
from app.services.llm import invalidate_llm_client_cache
invalidate_llm_client_cache()
invalidate_parser_plugin_cache()
invalidate_dedup_strategy_cache()
yield
invalidate_llm_client_cache()
invalidate_parser_plugin_cache()
invalidate_dedup_strategy_cache()
+271
View File
@@ -0,0 +1,271 @@
"""文本去重策略单元测试:none / sha256 / simhash 三种策略 lookup+record + 工厂
使用内存版 FakeRedis,不连真实 Redis。
"""
from typing import Any
from unittest.mock import AsyncMock
import pytest
from app.core import runtime_settings as rs
from app.core.dedup import (
DEDUP_KEY_PREFIX,
NoopDedupStrategy,
Sha256DedupStrategy,
SimhashDedupStrategy,
_hamming_distance,
_simhash,
compute_text_hash,
get_dedup_strategy,
invalidate_dedup_strategy_cache,
)
class FakeRedis:
"""内存版 Redis:实现 get_json/set_json,可记录所有写入"""
def __init__(self) -> None:
self.store: dict[str, dict[str, Any]] = {}
self.writes: list[tuple[str, dict[str, Any], int | None]] = []
async def get_json(self, key: str) -> dict[str, Any] | None:
return self.store.get(key)
async def set_json(self, key: str, value: dict[str, Any], ttl: int | None = None) -> bool:
self.writes.append((key, value, ttl))
self.store[key] = value
return True
@pytest.fixture
def isolated_settings_path(tmp_path, monkeypatch: pytest.MonkeyPatch):
path = tmp_path / "runtime_settings.json"
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
rs._runtime_settings = None
invalidate_dedup_strategy_cache()
yield path
rs._runtime_settings = None
invalidate_dedup_strategy_cache()
# ---------------------------------------------------------------------------- #
# 工具函数
# ---------------------------------------------------------------------------- #
class TestSimhashUtil:
def test_empty_text_returns_zero(self):
assert _simhash("") == 0
def test_same_text_same_fingerprint(self):
assert _simhash("同一段文本") == _simhash("同一段文本")
def test_different_text_different_fingerprint(self):
assert _simhash("文本A") != _simhash("文本B是完全不同的内容")
def test_hamming_distance_zero_for_same(self):
fp = _simhash("hello")
assert _hamming_distance(fp, fp) == 0
def test_hamming_distance_count(self):
# 0b001 vs 0b100 → 两个位不同
assert _hamming_distance(0b001, 0b100) == 2
def test_compute_text_hash_is_sha256_hex(self):
import hashlib
text = "abc"
assert compute_text_hash(text) == hashlib.sha256(text.encode("utf-8")).hexdigest()
# ---------------------------------------------------------------------------- #
# NoopDedupStrategy
# ---------------------------------------------------------------------------- #
class TestNoopStrategy:
async def test_lookup_always_none(self):
s = NoopDedupStrategy()
assert await s.lookup("any") is None
async def test_record_does_nothing(self):
s = NoopDedupStrategy()
await s.record("any", {"x": 1}) # 不抛异常即可
# ---------------------------------------------------------------------------- #
# Sha256DedupStrategy
# ---------------------------------------------------------------------------- #
class TestSha256Strategy:
def test_key_format(self):
s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100)
key = s._key("text")
assert key.startswith(f"{DEDUP_KEY_PREFIX}sha256:")
# 后缀是 64 位 sha256 hex
suffix = key[len(f"{DEDUP_KEY_PREFIX}sha256:"):]
assert len(suffix) == 64
async def test_lookup_miss_when_empty(self):
s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100)
assert await s.lookup("text") is None
async def test_record_then_lookup_hit(self):
redis = FakeRedis()
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
await s.record("text", {"document_id": "doc-1"})
hit = await s.lookup("text")
assert hit is not None
assert hit["document_id"] == "doc-1"
async def test_record_uses_ttl(self):
redis = FakeRedis()
s = Sha256DedupStrategy(redis=redis, ttl_seconds=42)
await s.record("text", {"x": 1})
# 检查写入 Redis 时使用的 TTL
key = s._key("text")
writes = [(k, v, ttl) for k, v, ttl in redis.writes if k == key]
assert writes
assert writes[0][2] == 42
async def test_lookup_redis_error_returns_none(self):
"""Redis 抛错时降级为未命中"""
redis = AsyncMock()
redis.get_json = AsyncMock(side_effect=RuntimeError("redis down"))
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
assert await s.lookup("text") is None
async def test_record_redis_error_swallows(self):
"""Redis 写入抛错时不向上抛"""
redis = AsyncMock()
redis.set_json = AsyncMock(side_effect=RuntimeError("redis down"))
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
await s.record("text", {"x": 1}) # 不抛
# ---------------------------------------------------------------------------- #
# SimhashDedupStrategy
# ---------------------------------------------------------------------------- #
class TestSimhashStrategy:
async def test_lookup_miss_when_empty_index(self):
s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=3)
assert await s.lookup("text") is None
async def test_record_then_lookup_identical_text(self):
"""相同文本 simhash 相同,距离 0 <= 阈值,命中"""
redis = FakeRedis()
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
await s.record("一段文本", {"document_id": "d1"})
hit = await s.lookup("一段文本")
assert hit is not None
assert hit["document_id"] == "d1"
async def test_lookup_similar_text_within_threshold(self):
"""相似文本(海明距离 <= 阈值)也命中"""
redis = FakeRedis()
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=64) # 阈值放大确保命中
await s.record("原文档文本内容示例", {"document_id": "d1"})
# 改一个字符
hit = await s.lookup("原文档文本内容示例改")
assert hit is not None
async def test_lookup_different_text_below_threshold_misses(self):
"""完全不同文本距离大,未命中"""
redis = FakeRedis()
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
await s.record("完全不同的第一种文本内容用于测试", {"document_id": "d1"})
hit = await s.lookup("另一段毫不相关的内容用于测试去重逻辑")
# 距离应该比较大;若碰巧小于阈值(小概率),改大文本差异
if hit is not None:
# 极小概率命中,放宽断言:至少 record 已写入
assert "document_id" in hit
else:
assert hit is None
async def test_record_updates_index(self):
"""record 把新 simhash 加入索引"""
redis = FakeRedis()
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
await s.record("文本A", {"x": 1})
await s.record("文本B完全不同", {"x": 2})
index = await redis.get_json(SimhashDedupStrategy.INDEX_KEY)
assert index is not None
assert len(index["entries"]) == 2
async def test_threshold_clamped(self):
"""threshold 超出 0~64 范围被夹紧"""
s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=999)
assert s._threshold == 64
s2 = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=-5)
assert s2._threshold == 0
async def test_lookup_redis_error_returns_none(self):
redis = AsyncMock()
redis.get_json = AsyncMock(side_effect=RuntimeError("redis down"))
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
assert await s.lookup("text") is None
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
class TestFactory:
def test_none_strategy_when_redis_none(self, isolated_settings_path):
"""redis=None 时无论配置如何,都返回 NoopDedupStrategy"""
s = get_dedup_strategy(None)
assert isinstance(s, NoopDedupStrategy)
def test_none_strategy_when_config_none(self, isolated_settings_path):
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
s = get_dedup_strategy(FakeRedis())
assert isinstance(s, NoopDedupStrategy)
def test_sha256_strategy(self, isolated_settings_path):
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
s = get_dedup_strategy(FakeRedis())
assert isinstance(s, Sha256DedupStrategy)
def test_simhash_strategy(self, isolated_settings_path):
rs.update_runtime_settings(
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
)
s = get_dedup_strategy(FakeRedis())
assert isinstance(s, SimhashDedupStrategy)
assert s._threshold == 5
def test_cache_same_signature_returns_same_instance(self, isolated_settings_path):
"""配置签名相同时复用单例"""
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
s1 = get_dedup_strategy(FakeRedis())
s2 = get_dedup_strategy(FakeRedis())
assert s1 is s2
def test_cache_invalidated_on_signature_change(self, isolated_settings_path):
"""配置签名变化时重建单例"""
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
s1 = get_dedup_strategy(FakeRedis())
rs.update_runtime_settings({"dedup": {"strategy": "simhash"}})
s2 = get_dedup_strategy(FakeRedis())
assert s1 is not s2
assert isinstance(s2, SimhashDedupStrategy)
def test_invalidate_cache_clears(self, isolated_settings_path):
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
s1 = get_dedup_strategy(FakeRedis())
invalidate_dedup_strategy_cache()
s2 = get_dedup_strategy(FakeRedis())
assert s1 is not s2
def test_ttl_change_rebuilds(self, isolated_settings_path):
"""ttl 变化也触发重建(签名包含 ttl)"""
rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 100}})
s1 = get_dedup_strategy(FakeRedis())
rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 200}})
s2 = get_dedup_strategy(FakeRedis())
assert s1 is not s2
+8 -2
View File
@@ -266,9 +266,15 @@ def test_upload_rejects_empty_text_after_parse(
def test_upload_rejects_corrupted_pdf(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""损坏的 PDFcode=1001message 含'文件解析失败',未提交任务"""
"""损坏的 PDFcode=1001message 含'文件解析失败''无法从文件提取文本',未提交任务
file_parser 插件化后:pypdf 失败被捕获并降级到 OCR;若 OCR 不可用/关闭,
最终返回空文本,由 upload 端点统一报 '无法从文件提取文本'
关闭 OCR 避免触发 rapidocr 模型下载拖慢测试。
"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
monkeypatch.setattr(document_module.settings, "pdf_ocr_enabled", False)
resp = client.post(
"/api/v1/documents/upload",
@@ -277,7 +283,7 @@ def test_upload_rejects_corrupted_pdf(
body = resp.json()
assert body["code"] == 1001
assert "文件解析失败" in body["message"]
assert "文件解析失败" in body["message"] or "无法从文件提取文本" in body["message"]
assert manager.submitted == []
+29 -11
View File
@@ -111,10 +111,17 @@ def test_parse_file_no_extension_raises() -> None:
parse_file("noext", b"text")
def test_parse_file_corrupted_pdf_raises() -> None:
"""损坏的 PDF 抛 ValueErrormessage 含'文件解析失败'"""
with pytest.raises(ValueError, match=r"文件解析失败"):
parse_file("bad.pdf", b"not a real pdf")
def test_parse_file_corrupted_pdf_returns_empty(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""损坏的 PDF:插件化后文本层提取失败被捕获,OCR 关闭时返回空字符串
原 test_parse_file_corrupted_pdf_raises 期望 ValueError,但插件化重构后
file_parser 设计为优雅降级(pypdf 失败 → OCR 兜底 → 都失败返回空),
不再向上抛异常。关闭 OCR 避免触发 rapidocr 模型下载拖慢测试。
"""
monkeypatch.setattr(settings, "pdf_ocr_enabled", False)
assert parse_file("bad.pdf", b"not a real pdf") == ""
def test_parse_file_empty_html_returns_empty_string() -> None:
@@ -152,12 +159,23 @@ def test_supported_extensions_contains_expected_set() -> None:
@pytest.fixture(autouse=True)
def _reset_ocr_state() -> Any:
"""每个 OCR 测试前后重置模块级 OCR 引擎状态,避免相互污染"""
saved_engine = fp_module._ocr_engine
saved_unavailable = fp_module._ocr_unavailable
"""每个 OCR 测试前后重置 RapidocrOcrEngine/TesseractOcrEngine 类级状态与插件缓存
file_parser 插件化后,模块级 _ocr_engine/_ocr_unavailable 已移除,
RapidocrOcrEngine 用类级字段 _engine/_unavailable 单例化。
测试前重置为干净状态(避免上个测试残留),测试后清理插件缓存。
"""
# 测试前:重置为干净状态
fp_module.RapidocrOcrEngine._engine = None
fp_module.RapidocrOcrEngine._unavailable = False
fp_module.TesseractOcrEngine._unavailable = False
fp_module._ocr_plugin_cache.clear()
yield
fp_module._ocr_engine = saved_engine
fp_module._ocr_unavailable = saved_unavailable
# 测试后:再次清理,避免污染后续非 OCR 测试
fp_module.RapidocrOcrEngine._engine = None
fp_module.RapidocrOcrEngine._unavailable = False
fp_module.TesseractOcrEngine._unavailable = False
fp_module._ocr_plugin_cache.clear()
class _FakeTextPage:
@@ -281,7 +299,7 @@ def test_parse_pdf_ocr_returns_empty_when_dependency_unavailable(
monkeypatch.setattr(settings, "pdf_ocr_enabled", True)
assert parse_file("scan.pdf", b"fake pdf bytes") == ""
assert fp_module._ocr_unavailable is True
assert fp_module.RapidocrOcrEngine._unavailable is True
def test_parse_pdf_ocr_runtime_exception_falls_back_to_empty(
@@ -324,4 +342,4 @@ def test_parse_pdf_text_layer_present_skips_ocr(
result = parse_file("text.pdf", b"fake pdf bytes")
assert result == "这是文本层的内容"
assert fp_module._ocr_engine is None
assert fp_module.RapidocrOcrEngine._engine is None
+5 -1
View File
@@ -227,7 +227,11 @@ async def test_get_falls_back_to_redis_then_none() -> None:
def _text_hash(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
"""构造与 Sha256DedupStrategy 一致的 dedup key
dedup 模块化后 key 形如 dedup:sha256:<sha256hex>(前缀 + 策略名 + hash)。
"""
return f"sha256:{hashlib.sha256(text.encode('utf-8')).hexdigest()}"
async def test_dedup_hit_reuses_old_doc_id_and_skips_pipeline() -> None:
+295
View File
@@ -0,0 +1,295 @@
"""LLM 工厂与客户端单元测试:mock httpx 验证 Ollama / OpenAICompatible 两条路径 + 缓存
通过 monkeypatch 替换 `httpx.AsyncClient` 为伪造的上下文管理器,避免真实网络请求。
"""
import httpx
import pytest
from unittest.mock import AsyncMock
from app.core import runtime_settings as rs
from app.services import llm as llm_mod
from app.services.llm import (
LLMClient,
OllamaLLMClient,
OpenAICompatibleLLMClient,
create_llm_client,
invalidate_llm_client_cache,
)
# ---------------------------------------------------------------------------- #
# httpx 伪造工具
# ---------------------------------------------------------------------------- #
class _FakeResponse:
def __init__(self, status_code: int = 200, json_data: dict | None = None) -> None:
self.status_code = status_code
self._json = json_data or {}
def raise_for_status(self) -> None:
if self.status_code >= 400:
raise httpx.HTTPStatusError(
f"HTTP {self.status_code}", request=httpx.Request("POST", "http://x"), response=self
)
def json(self) -> dict:
return self._json
class _FakeAsyncClient:
"""伪造 httpx.AsyncClient 上下文管理器
用法:把 _FakeAsyncClient.next_response 设为期望响应,构造后 post/get 返回该响应。
每次 __init__ 记录构造参数到 _FakeAsyncClient.last_kwargs。
"""
next_response: _FakeResponse | None = None
last_kwargs: dict | None = None
last_instances: list["_FakeAsyncClient"] = []
def __init__(self, *args, **kwargs) -> None:
self.post = AsyncMock()
self.get = AsyncMock()
if _FakeAsyncClient.next_response is not None:
self.post.return_value = _FakeAsyncClient.next_response
self.get.return_value = _FakeAsyncClient.next_response
_FakeAsyncClient.last_kwargs = kwargs
_FakeAsyncClient.last_instances.append(self)
async def __aenter__(self) -> "_FakeAsyncClient":
return self
async def __aexit__(self, *args) -> bool:
return False
@classmethod
def reset(cls) -> None:
cls.next_response = None
cls.last_kwargs = None
cls.last_instances = []
@pytest.fixture
def patch_httpx(monkeypatch: pytest.MonkeyPatch):
"""替换 httpx.AsyncClient 为伪造客户端"""
_FakeAsyncClient.reset()
monkeypatch.setattr(llm_mod.httpx, "AsyncClient", _FakeAsyncClient)
# ollama 模块内 OllamaClient 也用 httpx.AsyncClient
from app.services import ollama as ollama_mod
monkeypatch.setattr(ollama_mod.httpx, "AsyncClient", _FakeAsyncClient)
yield _FakeAsyncClient
_FakeAsyncClient.reset()
@pytest.fixture
def isolated_settings_path(tmp_path, monkeypatch: pytest.MonkeyPatch):
path = tmp_path / "runtime_settings.json"
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
rs._runtime_settings = None
invalidate_llm_client_cache()
yield path
rs._runtime_settings = None
invalidate_llm_client_cache()
# ---------------------------------------------------------------------------- #
# OllamaLLMClient
# ---------------------------------------------------------------------------- #
class TestOllamaLLMClient:
async def test_generate_returns_response_field(self, patch_httpx):
"""Ollama 响应:取 data['response'] 字段"""
patch_httpx.next_response = _FakeResponse(
json_data={"response": "你好世界", "model": "qwen2.5:1.5b"}
)
client = OllamaLLMClient(base_url="http://ollama:11434", model="qwen2.5:1.5b")
out = await client.generate("prompt")
assert out == "你好世界"
# 校验请求 url 与 payload
fake = patch_httpx.last_instances[-1]
fake.post.assert_awaited_once()
call_args = fake.post.await_args
assert call_args.args[0] == "http://ollama:11434/api/generate"
payload = call_args.kwargs["json"]
assert payload["model"] == "qwen2.5:1.5b"
assert payload["prompt"] == "prompt"
assert payload["stream"] is False
async def test_generate_json_mode_adds_format(self, patch_httpx):
patch_httpx.next_response = _FakeResponse(json_data={"response": "{}"})
client = OllamaLLMClient(base_url="http://o:11434", model="m")
await client.generate("p", json_mode=True)
payload = patch_httpx.last_instances[-1].post.await_args.kwargs["json"]
assert payload["format"] == "json"
async def test_is_available_true(self, patch_httpx):
patch_httpx.next_response = _FakeResponse(status_code=200)
client = OllamaLLMClient(base_url="http://o:11434", model="m")
assert await client.is_available() is True
async def test_is_available_false_on_http_error(self, monkeypatch):
"""httpx 抛 HTTPError 时 OllamaClient 内部捕获返回 FalseOllamaLLMClient 透传"""
class _RaisingClient:
async def __aenter__(self):
return self
async def __aexit__(self, *args):
return False
async def get(self, *args, **kwargs):
raise httpx.HTTPError("conn refused")
from app.services import ollama as ollama_mod
monkeypatch.setattr(ollama_mod.httpx, "AsyncClient", lambda *a, **kw: _RaisingClient())
client = OllamaLLMClient(base_url="http://o:11434", model="m")
assert await client.is_available() is False
# ---------------------------------------------------------------------------- #
# OpenAICompatibleLLMClient
# ---------------------------------------------------------------------------- #
class TestOpenAICompatibleClient:
async def test_generate_returns_choices_content(self, patch_httpx):
patch_httpx.next_response = _FakeResponse(
json_data={"choices": [{"message": {"content": "答案"}}]}
)
client = OpenAICompatibleLLMClient(
base_url="https://api.openai.com/v1", api_key="sk-x", model="gpt-4o-mini"
)
out = await client.generate("hello")
assert out == "答案"
fake = patch_httpx.last_instances[-1]
call_args = fake.post.await_args
assert call_args.args[0] == "https://api.openai.com/v1/chat/completions"
payload = call_args.kwargs["json"]
assert payload["model"] == "gpt-4o-mini"
assert payload["messages"] == [{"role": "user", "content": "hello"}]
assert payload["stream"] is False
# Authorization header
headers = call_args.kwargs["headers"]
assert headers["Authorization"] == "Bearer sk-x"
async def test_generate_json_mode_adds_response_format(self, patch_httpx):
patch_httpx.next_response = _FakeResponse(
json_data={"choices": [{"message": {"content": "{}"}}]}
)
client = OpenAICompatibleLLMClient(
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
)
await client.generate("p", json_mode=True)
payload = patch_httpx.last_instances[-1].post.await_args.kwargs["json"]
assert payload["response_format"] == {"type": "json_object"}
async def test_generate_empty_choices_returns_empty(self, patch_httpx):
patch_httpx.next_response = _FakeResponse(json_data={"choices": []})
client = OpenAICompatibleLLMClient(
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
)
assert await client.generate("p") == ""
async def test_is_available_true(self, patch_httpx):
patch_httpx.next_response = _FakeResponse(status_code=200)
client = OpenAICompatibleLLMClient(
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
)
assert await client.is_available() is True
async def test_is_available_false_on_http_error(self, monkeypatch):
"""httpx 抛 HTTPError 时返回 False"""
client = OpenAICompatibleLLMClient(
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
)
class _Raiser:
async def __aenter__(self):
raise httpx.HTTPError("conn refused")
async def __aexit__(self, *args):
return False
monkeypatch.setattr(llm_mod.httpx, "AsyncClient", lambda *a, **kw: _Raiser())
assert await client.is_available() is False
# ---------------------------------------------------------------------------- #
# 工厂
# ---------------------------------------------------------------------------- #
class TestFactory:
def test_ollama_provider_returns_ollama_client(self, isolated_settings_path):
"""runtime_settings 默认 provider=ollama,工厂返回 OllamaLLMClient"""
# 确保配置走 ollama(默认即 ollama
client = create_llm_client("summarize", use_cache=False)
assert isinstance(client, OllamaLLMClient)
def test_openai_compatible_provider_returns_openai_client(
self, isolated_settings_path
):
rs.update_runtime_settings(
{"models": {"query": {"provider": "openai_compatible", "api_key": "sk-x", "model": "gpt-4o-mini"}}}
)
client = create_llm_client("query", use_cache=False)
assert isinstance(client, OpenAICompatibleLLMClient)
assert client.model == "gpt-4o-mini"
assert client.api_key == "sk-x"
def test_openai_compatible_falls_back_to_settings(self, isolated_settings_path):
"""openai_compatible 时 base_url/api_key 为空用 settings 默认;model 为空用 gpt-4o-mini"""
# 显式置空 model 以触发工厂默认值(env fallback 会预填 ollama_model
rs.update_runtime_settings(
{"models": {"classify": {"provider": "openai_compatible", "model": ""}}}
)
from app.config import settings
client = create_llm_client("classify", use_cache=False)
assert isinstance(client, OpenAICompatibleLLMClient)
assert client.api_key == settings.openai_api_key
assert client.model == "gpt-4o-mini" # openai_compatible 默认模型
def test_cache_returns_same_instance(self, isolated_settings_path):
"""use_cache=True 时同 purpose 复用单例"""
c1 = create_llm_client("summarize")
c2 = create_llm_client("summarize")
assert c1 is c2
def test_cache_different_purposes_different_instances(self, isolated_settings_path):
c1 = create_llm_client("summarize")
c2 = create_llm_client("query")
assert c1 is not c2
def test_no_cache_returns_new_instance_each_call(self, isolated_settings_path):
c1 = create_llm_client("summarize", use_cache=False)
c2 = create_llm_client("summarize", use_cache=False)
assert c1 is not c2
def test_invalidate_clears_cache(self, isolated_settings_path):
c1 = create_llm_client("summarize")
invalidate_llm_client_cache()
c2 = create_llm_client("summarize")
assert c1 is not c2
def test_invalidate_single_purpose(self, isolated_settings_path):
c_sum = create_llm_client("summarize")
c_qry = create_llm_client("query")
invalidate_llm_client_cache("summarize")
# summarize 已清,query 未清
assert create_llm_client("summarize") is not c_sum
assert create_llm_client("query") is c_qry
def test_client_implements_protocol(self, isolated_settings_path):
"""OllamaLLMClient 与 OpenAICompatibleLLMClient 都满足 LLMClient Protocol"""
c1 = create_llm_client("summarize", use_cache=False)
assert isinstance(c1, LLMClient)
rs.update_runtime_settings(
{"models": {"summarize": {"provider": "openai_compatible", "api_key": "k"}}}
)
c2 = create_llm_client("summarize", use_cache=False)
assert isinstance(c2, LLMClient)
+254
View File
@@ -0,0 +1,254 @@
"""RuntimeSettings 单元测试:默认值 / 加载 / 保存 / 部分更新 / 重置 / 单例
每个测试通过 monkeypatch 把 RUNTIME_SETTINGS_PATH 指向独立临时文件,并在前后
重置模块级 `_runtime_settings` 单例,避免跨测试串扰。
"""
import json
from pathlib import Path
import pytest
from app.core import runtime_settings as rs
@pytest.fixture
def isolated_settings_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""每个测试独立持久化路径,并在前后清空模块级单例"""
path = tmp_path / "runtime_settings.json"
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
# 重置单例,强制下次 get_runtime_settings 重新加载
rs._runtime_settings = None
yield path
rs._runtime_settings = None
# ---------------------------------------------------------------------------- #
# 默认值
# ---------------------------------------------------------------------------- #
class TestDefaults:
def test_default_models_provider_is_ollama(self):
cfg = rs.RuntimeSettings()
assert cfg.models.summarize.provider == "ollama"
assert cfg.models.query.provider == "ollama"
assert cfg.models.classify.provider == "ollama"
def test_default_parsers_plugins(self):
cfg = rs.RuntimeSettings()
assert cfg.parsers.ocr.plugin == "rapidocr"
assert cfg.parsers.pdf.plugin == "pypdf"
assert cfg.parsers.docx.plugin == "python_docx"
def test_default_dedup(self):
cfg = rs.RuntimeSettings()
assert cfg.dedup.strategy == "sha256"
assert cfg.dedup.simhash_threshold == 3
assert cfg.dedup.ttl_seconds == 86400
def test_default_with_env_fallback_uses_ollama_env(self):
"""无持久化文件时,base_url/model 取自 settings.ollama_*"""
cfg = rs._default_with_env_fallback()
from app.config import settings
assert cfg.models.summarize.base_url == settings.ollama_base_url
assert cfg.models.summarize.model == settings.ollama_model
assert cfg.models.query.base_url == settings.ollama_base_url
assert cfg.models.classify.model == settings.ollama_model
# ---------------------------------------------------------------------------- #
# 加载
# ---------------------------------------------------------------------------- #
class TestLoad:
def test_missing_file_returns_env_fallback(self, isolated_settings_path: Path):
"""文件不存在时回退到带 env 兜底的默认值(不抛异常)"""
assert not isolated_settings_path.exists()
cfg = rs.load_runtime_settings()
# base_url 应来自 env fallback
from app.config import settings
assert cfg.models.summarize.base_url == settings.ollama_base_url
def test_valid_file_parsed(self, isolated_settings_path: Path):
isolated_settings_path.write_text(
json.dumps(
{
"models": {
"summarize": {"provider": "openai_compatible", "model": "gpt-4o-mini", "api_key": "k"}
},
"dedup": {"strategy": "simhash", "simhash_threshold": 5},
}
),
encoding="utf-8",
)
cfg = rs.load_runtime_settings()
assert cfg.models.summarize.provider == "openai_compatible"
assert cfg.models.summarize.model == "gpt-4o-mini"
assert cfg.models.summarize.api_key == "k"
assert cfg.dedup.strategy == "simhash"
assert cfg.dedup.simhash_threshold == 5
# 未指定的字段保留默认
assert cfg.dedup.ttl_seconds == 86400
assert cfg.models.query.provider == "ollama"
def test_corrupted_file_falls_back(self, isolated_settings_path: Path):
isolated_settings_path.write_text("not-json{", encoding="utf-8")
cfg = rs.load_runtime_settings()
# 回退到默认(ollama provider
assert cfg.models.summarize.provider == "ollama"
def test_invalid_values_falls_back(self, isolated_settings_path: Path):
"""字段值非法(如未知 provider)时整体回退默认"""
isolated_settings_path.write_text(
json.dumps({"models": {"summarize": {"provider": "unknown_provider"}}}),
encoding="utf-8",
)
cfg = rs.load_runtime_settings()
assert cfg.models.summarize.provider == "ollama" # 回退默认
# ---------------------------------------------------------------------------- #
# 保存
# ---------------------------------------------------------------------------- #
class TestSave:
def test_save_writes_valid_json(self, isolated_settings_path: Path):
cfg = rs.RuntimeSettings()
cfg.dedup.strategy = "simhash"
rs.save_runtime_settings(cfg)
assert isolated_settings_path.exists()
data = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
assert data["dedup"]["strategy"] == "simhash"
def test_save_creates_parent_dir(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
path = tmp_path / "nested" / "deep" / "runtime_settings.json"
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
rs.save_runtime_settings(rs.RuntimeSettings())
assert path.exists()
def test_save_atomic_no_tmp_left(self, isolated_settings_path: Path):
"""保存后同目录无残留 .tmp 临时文件"""
rs.save_runtime_settings(rs.RuntimeSettings())
tmps = list(isolated_settings_path.parent.glob(".runtime_settings.*.tmp"))
assert tmps == []
# ---------------------------------------------------------------------------- #
# 单例 + reload
# ---------------------------------------------------------------------------- #
class TestSingleton:
def test_get_returns_singleton(self, isolated_settings_path: Path):
cfg1 = rs.get_runtime_settings()
cfg2 = rs.get_runtime_settings()
assert cfg1 is cfg2
def test_reload_rereads_disk(self, isolated_settings_path: Path):
"""reload 强制重新读盘,单例替换为新对象"""
cfg1 = rs.get_runtime_settings()
# 直接改盘上文件
isolated_settings_path.write_text(
json.dumps({"dedup": {"strategy": "none"}}), encoding="utf-8"
)
cfg2 = rs.reload_runtime_settings()
assert cfg2 is not cfg1
assert cfg2.dedup.strategy == "none"
# ---------------------------------------------------------------------------- #
# 部分更新(深合并)
# ---------------------------------------------------------------------------- #
class TestUpdate:
def test_partial_update_models_summarize(self, isolated_settings_path: Path):
rs.get_runtime_settings() # 初始化单例
new_cfg = rs.update_runtime_settings(
{"models": {"summarize": {"model": "qwen2.5:3b"}}}
)
assert new_cfg.models.summarize.model == "qwen2.5:3b"
# 其他字段保留
assert new_cfg.models.summarize.provider == "ollama"
assert new_cfg.models.query.provider == "ollama"
def test_partial_update_dedup(self, isolated_settings_path: Path):
rs.get_runtime_settings()
new_cfg = rs.update_runtime_settings(
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
)
assert new_cfg.dedup.strategy == "simhash"
assert new_cfg.dedup.simhash_threshold == 5
# ttl 未在 patch 中,保留默认
assert new_cfg.dedup.ttl_seconds == 86400
def test_update_persists_to_disk(self, isolated_settings_path: Path):
rs.get_runtime_settings()
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
data = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
assert data["dedup"]["strategy"] == "none"
def test_update_replaces_singleton(self, isolated_settings_path: Path):
old = rs.get_runtime_settings()
new = rs.update_runtime_settings({"dedup": {"strategy": "none"}})
assert new is not old
# 后续 get 拿到的是新单例
assert rs.get_runtime_settings() is new
def test_update_empty_patch_keeps_all(self, isolated_settings_path: Path):
"""空 patch 不改变任何字段"""
rs.get_runtime_settings()
new_cfg = rs.update_runtime_settings({})
assert new_cfg.dedup.strategy == "sha256"
# ---------------------------------------------------------------------------- #
# 重置
# ---------------------------------------------------------------------------- #
class TestReset:
def test_reset_returns_defaults(self, isolated_settings_path: Path):
# 先污染
rs.get_runtime_settings()
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
assert rs.get_runtime_settings().dedup.strategy == "none"
# 重置
cfg = rs.reset_runtime_settings()
assert cfg.dedup.strategy == "sha256"
assert cfg.dedup.simhash_threshold == 3
assert cfg.parsers.ocr.plugin == "rapidocr"
def test_reset_persists_to_disk(self, isolated_settings_path: Path):
rs.get_runtime_settings()
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
rs.reset_runtime_settings()
data = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
assert data["dedup"]["strategy"] == "sha256"
# ---------------------------------------------------------------------------- #
# 深合并工具函数
# ---------------------------------------------------------------------------- #
class TestDeepMerge:
def test_nested_dict_merged(self):
target = {"a": {"b": 1, "c": 2}, "d": 3}
rs._deep_merge(target, {"a": {"b": 10}})
assert target == {"a": {"b": 10, "c": 2}, "d": 3}
def test_non_dict_overrides(self):
target = {"a": {"b": 1}}
rs._deep_merge(target, {"a": 99})
assert target == {"a": 99}
def test_new_key_added(self):
target = {"a": 1}
rs._deep_merge(target, {"b": 2})
assert target == {"a": 1, "b": 2}