feat: 完成全量功能开发,包括前端管理后台与后端服务优化
此提交实现了完整的知识库管理系统: 1. 新增Vue3 + Antd Vue前端管理后台,包含登录、文档管理、检索、类目设置等完整页面 2. 重构后端LLM调用抽象层,支持Ollama与OpenAI兼容服务动态切换 3. 调整默认嵌入模型配置为本地bge-m3模式 4. 优化入库任务去重逻辑与缓存清理机制 5. 完善Docker镜像构建与docker-compose部署配置 6. 修复多项测试用例与兼容性问题 7. 新增运行时配置API,支持动态调整系统参数
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
.git
|
||||
.venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
.pytest_cache
|
||||
htmlcov
|
||||
.coverage
|
||||
data
|
||||
logs
|
||||
*.log
|
||||
.env
|
||||
.env.local
|
||||
docker-compose.override.yml
|
||||
frontend/node_modules
|
||||
frontend/dist
|
||||
app/static/admin
|
||||
@@ -6,6 +6,9 @@ dist/
|
||||
build/
|
||||
.venv/
|
||||
|
||||
# Frontend
|
||||
node_modules/
|
||||
|
||||
# Env
|
||||
.env
|
||||
.env.local
|
||||
|
||||
+17
@@ -1,3 +1,17 @@
|
||||
# ---------- Stage 1: 前端构建 ----------
|
||||
FROM node:20-slim AS frontend-builder
|
||||
|
||||
WORKDIR /frontend
|
||||
|
||||
# 依赖层(利用 Docker 层缓存)
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
# 源码 + 构建
|
||||
COPY frontend/ ./
|
||||
RUN npm run build
|
||||
|
||||
# ---------- Stage 2: Python 后端 ----------
|
||||
FROM python:3.12-slim AS base
|
||||
|
||||
WORKDIR /app
|
||||
@@ -13,6 +27,9 @@ RUN uv sync --frozen --no-dev
|
||||
COPY app/ app/
|
||||
COPY scripts/ scripts/
|
||||
|
||||
# 前端构建产物 → 后端静态目录(SPA 由 /admin 路由服务)
|
||||
COPY --from=frontend-builder /frontend/dist /app/app/static/admin
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""运行时配置 API:GET/PUT /api/v1/settings、GET /api/v1/settings/schema、POST /api/v1/settings/reset
|
||||
|
||||
GET 任何登录用户可读;PUT/reset 需 admin。PUT 后会清空 LLM 客户端 / 解析插件 /
|
||||
去重策略三处进程级缓存,使新配置立即对后续请求生效。
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.api.response import ApiError, ok
|
||||
from app.core.auth import AuthUser, get_current_user, require_admin
|
||||
from app.core.dedup import invalidate_dedup_strategy_cache
|
||||
from app.core.file_parser import (
|
||||
list_docx_plugins,
|
||||
list_ocr_plugins,
|
||||
list_pdf_plugins,
|
||||
invalidate_parser_plugin_cache,
|
||||
)
|
||||
from app.core.runtime_settings import (
|
||||
RuntimeSettings,
|
||||
get_runtime_settings,
|
||||
reset_runtime_settings,
|
||||
update_runtime_settings,
|
||||
)
|
||||
from app.services.llm import invalidate_llm_client_cache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["settings"])
|
||||
|
||||
|
||||
class SettingsUpdateRequest(BaseModel):
|
||||
"""Settings PATCH body:任意子树可缺省,缺省字段保留原值
|
||||
|
||||
例:{"models": {"summarize": {"model": "qwen2.5:3b"}}}、
|
||||
{"parsers": {"ocr": {"plugin": "tesseract"}}}、
|
||||
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
|
||||
"""
|
||||
|
||||
models: dict[str, Any] | None = None
|
||||
parsers: dict[str, Any] | None = None
|
||||
dedup: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@router.get("/settings")
|
||||
async def get_settings(user: AuthUser = Depends(get_current_user)) -> dict[str, Any]:
|
||||
"""返回当前 RuntimeSettings(任何登录用户可读)"""
|
||||
cfg = get_runtime_settings()
|
||||
return ok(cfg.model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.put("/settings")
|
||||
async def update_settings(
|
||||
body: SettingsUpdateRequest,
|
||||
user: AuthUser = Depends(require_admin),
|
||||
) -> dict[str, Any]:
|
||||
"""部分更新 RuntimeSettings(仅 admin)
|
||||
|
||||
更新成功后清空 LLM 客户端 / 解析插件 / 去重策略三处进程级缓存,
|
||||
使新配置立即对后续请求生效。
|
||||
"""
|
||||
patch = body.model_dump(exclude_none=True)
|
||||
if not patch:
|
||||
raise ApiError(1001, "请求体为空,未提供任何待更新字段")
|
||||
|
||||
try:
|
||||
cfg = update_runtime_settings(patch)
|
||||
except Exception as exc:
|
||||
logger.error("RuntimeSettings 更新失败", error=str(exc), exc_info=True)
|
||||
raise ApiError(2000, f"配置更新失败: {exc}") from exc
|
||||
|
||||
# 清缓存:让后续读取拿到新配置
|
||||
invalidate_llm_client_cache()
|
||||
invalidate_parser_plugin_cache()
|
||||
invalidate_dedup_strategy_cache()
|
||||
logger.info("RuntimeSettings 已更新并清理缓存", operator=user.username)
|
||||
return ok(cfg.model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.get("/settings/schema")
|
||||
async def get_settings_schema(
|
||||
user: AuthUser = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
"""返回可选插件与策略列表(前端 Settings 页渲染选项用)"""
|
||||
return ok(
|
||||
{
|
||||
"llm_providers": ["ollama", "openai_compatible"],
|
||||
"pdf_plugins": list_pdf_plugins(),
|
||||
"docx_plugins": list_docx_plugins(),
|
||||
"ocr_plugins": list_ocr_plugins(),
|
||||
"dedup_strategies": ["none", "sha256", "simhash"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/settings/reset")
|
||||
async def reset_settings(
|
||||
user: AuthUser = Depends(require_admin),
|
||||
) -> dict[str, Any]:
|
||||
"""重置 RuntimeSettings 为默认值(仅 admin),同时清缓存"""
|
||||
try:
|
||||
cfg = reset_runtime_settings()
|
||||
except Exception as exc:
|
||||
logger.error("RuntimeSettings 重置失败", error=str(exc), exc_info=True)
|
||||
raise ApiError(2000, f"配置重置失败: {exc}") from exc
|
||||
|
||||
invalidate_llm_client_cache()
|
||||
invalidate_parser_plugin_cache()
|
||||
invalidate_dedup_strategy_cache()
|
||||
logger.info("RuntimeSettings 已重置为默认值", operator=user.username)
|
||||
return ok(cfg.model_dump(mode="json"))
|
||||
+3
-2
@@ -9,11 +9,12 @@ class Settings(BaseSettings):
|
||||
log_level: str = "info"
|
||||
|
||||
# 嵌入模型
|
||||
embedding_provider: str = "openai" # openai | local
|
||||
embedding_provider: str = "local" # openai | local
|
||||
openai_api_key: str = ""
|
||||
openai_base_url: str = "https://api.openai.com/v1"
|
||||
embedding_model: str = "text-embedding-3-small"
|
||||
embedding_dimension: int = 1536
|
||||
# bge-m3(本地 Ollama 嵌入)维度为 1024;切换 openai provider 时需同步改为 1536
|
||||
embedding_dimension: int = 1024
|
||||
|
||||
# Ollama 本地模型(用于文档三级总结)
|
||||
ollama_base_url: str = "http://localhost:11434"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""文档分类器
|
||||
|
||||
入库链路第二步:基于 L1 总结,用 Ollama 小模型将文档判定为 taxonomy 中的
|
||||
入库链路第二步:基于 L1 总结,用 LLM(Ollama 或 OpenAI 兼容服务,由
|
||||
runtime_settings.models.classify 决定)将文档判定为 taxonomy 中的
|
||||
主类目 + 附加标签:
|
||||
- LLM 输出解析失败 / 类目名不在 taxonomy → 归 uncategorized(confidence=0.0)
|
||||
- 置信度低于阈值 → 主类目归 uncategorized,候选类目名保留进 tags(软召回用)
|
||||
@@ -13,7 +14,7 @@ import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.models.knowledge import UNCATEGORIZED, CategoryResult, TaxonomyCategory, load_taxonomy
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
@@ -47,8 +48,10 @@ def _extract_json(raw: str) -> dict | None:
|
||||
class Classifier:
|
||||
"""文档分类器:将 L1 总结判定为 taxonomy 主类目 + 附加标签"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
def __init__(self, ollama: LLMClient | None = None, taxonomy: list[TaxonomyCategory] | None = None) -> None:
|
||||
# 默认按 runtime_settings.models.classify 选择 LLM 实现;
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("classify")
|
||||
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
|
||||
# 合法类目名集合(含 uncategorized)
|
||||
self._valid_names = {c.name for c in self.taxonomy}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""文本去重策略工厂
|
||||
|
||||
支持三种策略:
|
||||
- none:关闭去重,每次都跑完整流水线
|
||||
- sha256:精确匹配(hash 完全相同才算重复)
|
||||
- simhash:近似匹配(海明距离 <= 阈值视为重复)
|
||||
|
||||
每种策略实现 DedupStrategy 协议:lookup(text) -> dict|None、record(text, result) -> None。
|
||||
工厂 get_dedup_strategy() 根据 runtime_settings.dedup 选择实现。
|
||||
所有策略共享同一 Redis key 前缀,但 key 后缀按策略区分避免冲突。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
import structlog
|
||||
|
||||
from app.core.runtime_settings import get_runtime_settings
|
||||
from app.services.redis import RedisCache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# Redis 去重 key 前缀,按策略分桶
|
||||
_DEDUP_KEY_PREFIX = "dedup:"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DedupStrategy(Protocol):
|
||||
"""去重策略协议"""
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
"""查询文本是否已入库;命中返回旧 IngestionResult(dict),未命中返回 None"""
|
||||
...
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
"""记录文本已入库,供后续命中复用"""
|
||||
...
|
||||
|
||||
|
||||
class NoopDedupStrategy:
|
||||
"""关闭去重:永远不命中,也不记录"""
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class Sha256DedupStrategy:
|
||||
"""SHA256 精确去重:完全相同文本才算重复"""
|
||||
|
||||
def __init__(self, redis: RedisCache, ttl_seconds: int) -> None:
|
||||
self._redis = redis
|
||||
self._ttl = ttl_seconds
|
||||
|
||||
def _key(self, text: str) -> str:
|
||||
h = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
return f"{_DEDUP_KEY_PREFIX}sha256:{h}"
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
return await self._redis.get_json(self._key(text))
|
||||
except Exception:
|
||||
logger.warning("sha256 去重查询失败,降级未命中", exc_info=True)
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._redis.set_json(self._key(text), result_dict, ttl=self._ttl)
|
||||
except Exception:
|
||||
logger.warning("sha256 去重记录写入失败", exc_info=True)
|
||||
|
||||
|
||||
# SimHash 实现(64 位)
|
||||
_MASK_64 = (1 << 64) - 1
|
||||
|
||||
|
||||
def _simhash(text: str, token_size: int = 4) -> int:
|
||||
"""计算文本 64 位 simhash
|
||||
|
||||
简化实现:按 token_size 字符滑窗分词,每段 md5 → 128 位 → 取低 64 位作为 hash,
|
||||
逐位加权(hash 该位为 1 则 +1,为 0 则 -1),最终符号位定 1/0。
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
tokens = [text[i : i + token_size] for i in range(0, len(text), token_size)]
|
||||
weights = [1] * 64
|
||||
vec = [0] * 64
|
||||
for token in tokens:
|
||||
h = int(hashlib.md5(token.encode("utf-8")).hexdigest(), 16) & _MASK_64
|
||||
for i in range(64):
|
||||
bit = (h >> i) & 1
|
||||
vec[i] += weights[i] if bit else -weights[i]
|
||||
fingerprint = 0
|
||||
for i in range(64):
|
||||
if vec[i] > 0:
|
||||
fingerprint |= (1 << i)
|
||||
return fingerprint
|
||||
|
||||
|
||||
def _hamming_distance(a: int, b: int) -> int:
|
||||
return bin((a ^ b) & _MASK_64).count("1")
|
||||
|
||||
|
||||
class SimhashDedupStrategy:
|
||||
"""SimHash 近似去重:海明距离 <= 阈值视为重复
|
||||
|
||||
Redis 中存所有已入库文本的 simhash(value 为 IngestionResult + simhash)。
|
||||
lookup 时遍历所有候选 simhash 算海明距离,找最近的一个 <= 阈值则命中。
|
||||
注:本实现为简化版,全表扫描,适合中小规模知识库;超大规模需换 LSH 索引。
|
||||
"""
|
||||
|
||||
INDEX_KEY = f"{_DEDUP_KEY_PREFIX}simhash:index" # list 形式存所有 simhash+key
|
||||
|
||||
def __init__(self, redis: RedisCache, ttl_seconds: int, threshold: int = 3) -> None:
|
||||
self._redis = redis
|
||||
self._ttl = ttl_seconds
|
||||
self._threshold = max(0, min(64, threshold))
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
fingerprint = _simhash(text)
|
||||
# 简化:扫描所有 simhash 记录找最近的
|
||||
# 用 set:dedup:simhash:fps 存所有 fingerprint,每个 fp 对应一个 record key
|
||||
# 这里用 list key 简化(适合小规模)
|
||||
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
|
||||
entries = index.get("entries", [])
|
||||
best_dist = self._threshold + 1
|
||||
best_key: str | None = None
|
||||
for entry in entries:
|
||||
fp = entry.get("fingerprint")
|
||||
key = entry.get("key")
|
||||
if fp is None or key is None:
|
||||
continue
|
||||
dist = _hamming_distance(fingerprint, int(fp))
|
||||
if dist <= self._threshold and dist < best_dist:
|
||||
best_dist = dist
|
||||
best_key = key
|
||||
if best_key is None:
|
||||
return None
|
||||
return await self._redis.get_json(best_key)
|
||||
except Exception:
|
||||
logger.warning("simhash 去重查询失败,降级未命中", exc_info=True)
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
try:
|
||||
fingerprint = _simhash(text)
|
||||
key = f"{_DEDUP_KEY_PREFIX}simhash:{fingerprint:016x}"
|
||||
# 1. 写记录
|
||||
await self._redis.set_json(key, result_dict, ttl=self._ttl)
|
||||
# 2. 更新索引
|
||||
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
|
||||
entries = index.get("entries", [])
|
||||
entries.append({"fingerprint": fingerprint, "key": key})
|
||||
await self._redis.set_json(self.INDEX_KEY, {"entries": entries}, ttl=self._ttl)
|
||||
except Exception:
|
||||
logger.warning("simhash 去重记录写入失败", exc_info=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_strategy_cache: DedupStrategy | None = None
|
||||
_strategy_signature: tuple[str, int, int] | None = None # (strategy, ttl, threshold)
|
||||
|
||||
|
||||
def get_dedup_strategy(redis: RedisCache | None) -> DedupStrategy:
|
||||
"""按 runtime_settings.dedup 构造去重策略
|
||||
|
||||
Redis 不可用时降级为 NoopDedupStrategy(关闭去重),不影响主流程。
|
||||
策略配置变更时自动重建单例。
|
||||
"""
|
||||
global _strategy_cache, _strategy_signature
|
||||
|
||||
rt = get_runtime_settings()
|
||||
sig = (rt.dedup.strategy, rt.dedup.ttl_seconds, rt.dedup.simhash_threshold)
|
||||
|
||||
if _strategy_cache is not None and _strategy_signature == sig:
|
||||
return _strategy_cache
|
||||
|
||||
if redis is None or rt.dedup.strategy == "none":
|
||||
strategy: DedupStrategy = NoopDedupStrategy()
|
||||
elif rt.dedup.strategy == "sha256":
|
||||
strategy = Sha256DedupStrategy(redis=redis, ttl_seconds=rt.dedup.ttl_seconds)
|
||||
elif rt.dedup.strategy == "simhash":
|
||||
strategy = SimhashDedupStrategy(
|
||||
redis=redis, ttl_seconds=rt.dedup.ttl_seconds, threshold=rt.dedup.simhash_threshold
|
||||
)
|
||||
else:
|
||||
logger.warning("未知去重策略,降级为 noop", requested=rt.dedup.strategy)
|
||||
strategy = NoopDedupStrategy()
|
||||
|
||||
_strategy_cache = strategy
|
||||
_strategy_signature = sig
|
||||
return strategy
|
||||
|
||||
|
||||
def invalidate_dedup_strategy_cache() -> None:
|
||||
"""清除策略缓存(runtime_settings 更新后调用)"""
|
||||
global _strategy_cache, _strategy_signature
|
||||
_strategy_cache = None
|
||||
_strategy_signature = None
|
||||
|
||||
|
||||
def compute_text_hash(text: str) -> str:
|
||||
"""计算文本哈希(兼容旧接口,sha256 策略下与 dedup key 一致)"""
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
# 兼容旧测试与外部引用:保留 DEDUP_KEY_PREFIX 导出
|
||||
DEDUP_KEY_PREFIX = _DEDUP_KEY_PREFIX
|
||||
+346
-87
@@ -1,14 +1,19 @@
|
||||
"""多格式文件文本提取:按扩展名分发到对应解析器
|
||||
"""多格式文件文本提取:按扩展名分发到对应解析器(插件化)
|
||||
|
||||
支持的扩展名:
|
||||
- .txt / .md:UTF-8 解码(errors="replace" 兜底)
|
||||
- .html / .htm:标准库 html.parser 剥离标签提取可见文本
|
||||
- .pdf:pypdf 逐页 extract_text 拼接;文本层为空(扫描件/图片型)时
|
||||
自动降级为 OCR(pypdfium2 渲染 + rapidocr-onnxruntime 识别)
|
||||
- .docx:python-docx 段落文本拼接(不含表格/页眉页脚)
|
||||
- .pdf:PDF 文本层提取插件(默认 pypdf);文本层为空(扫描件)时降级到 OCR 插件
|
||||
- .docx:DOCX 解析插件(默认 python-docx)
|
||||
|
||||
插件化设计:
|
||||
- PdfTextExtractor / DocxParser / OcrEngine 三个 Protocol
|
||||
- 每种插件类型有注册表 + 默认实现 + 工厂
|
||||
- 工厂根据 runtime_settings.parsers 选择具体插件
|
||||
- 未注册或依赖缺失时降级到默认插件并告警
|
||||
|
||||
未识别扩展名抛 ValueError("不支持的文件类型: {ext}");
|
||||
解析异常统一包装为 ValueError("文件解析失败: {detail}"),原异常链式保留。
|
||||
解析异常统一包装为 ValueError("文件解析失败: {detail}")。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,17 +22,19 @@ import io
|
||||
from collections.abc import Callable
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.core.runtime_settings import get_runtime_settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 模块级懒加载 OCR 引擎单例(首次调用时初始化,避免无扫描件场景白白下载模型)
|
||||
_ocr_engine: Any | None = None
|
||||
_ocr_unavailable: bool = False # 标记 OCR 依赖不可用,后续直接跳过避免重复尝试
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# HTML / Text 解析(无插件化需求,保留原实现)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _VisibleTextExtractor(HTMLParser):
|
||||
@@ -51,13 +58,11 @@ class _VisibleTextExtractor(HTMLParser):
|
||||
self._parts.append(data)
|
||||
|
||||
def get_text(self) -> str:
|
||||
# 块级标签间用空格连接,再折叠多余空白
|
||||
text = " ".join(self._parts)
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def _decode_html(content: bytes) -> str:
|
||||
"""HTML 内容解码:优先 utf-8(带 BOM),失败回退 latin-1"""
|
||||
try:
|
||||
return content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
@@ -65,108 +70,362 @@ def _decode_html(content: bytes) -> str:
|
||||
|
||||
|
||||
def _parse_text(content: bytes) -> str:
|
||||
"""UTF-8 解码(errors=replace 兜底),保留原字符"""
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _parse_html(content: bytes) -> str:
|
||||
"""HTML 剥离标签,保留可见文本"""
|
||||
parser = _VisibleTextExtractor()
|
||||
parser.feed(_decode_html(content))
|
||||
parser.close()
|
||||
return parser.get_text()
|
||||
|
||||
|
||||
def _parse_pdf(content: bytes) -> str:
|
||||
"""PDF 解析:优先 pypdf extract_text;文本层为空(扫描件)时降级 OCR
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 插件协议
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
OCR 流程:pypdfium2 渲染每页为 PIL Image → rapidocr-onnxruntime 识别 →
|
||||
拼接每页识别出的文本。受 settings.pdf_ocr_* 控制:开关、最大页数、DPI。
|
||||
OCR 依赖未安装或运行异常时降级返回空字符串(由上游 upload 端点拒绝入库)。
|
||||
|
||||
@runtime_checkable
|
||||
class PdfTextExtractor(Protocol):
|
||||
"""PDF 文本层提取插件协议"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
"""从 PDF 二进制内容提取文本层;无文本层返回空字符串"""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DocxParser(Protocol):
|
||||
"""DOCX 解析插件协议"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
"""从 DOCX 二进制内容提取段落文本"""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class OcrEngine(Protocol):
|
||||
"""OCR 引擎插件协议"""
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
"""对 PDF 跑 OCR,返回识别文本;失败返回空字符串"""
|
||||
...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 PDF 插件:pypdf
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PypdfTextExtractor:
|
||||
"""pypdf 文本层提取(默认)"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(io.BytesIO(content))
|
||||
parts: list[str] = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 备选 PDF 插件:pdfplumber(lazy import,依赖缺失时不可用)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PdfplumberTextExtractor:
|
||||
"""pdfplumber 文本层提取(备选,对复杂排版更友好)"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
import pdfplumber # type: ignore[import-untyped]
|
||||
|
||||
parts: list[str] = []
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
for page in pdf.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 DOCX 插件:python-docx
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PythonDocxParser:
|
||||
"""python-docx 段落提取(默认)"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
from docx import Document # type: ignore[import-untyped]
|
||||
|
||||
document = Document(io.BytesIO(content))
|
||||
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 OCR 插件:rapidocr-onnxruntime + pypdfium2
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class RapidocrOcrEngine:
|
||||
"""rapidocr-onnxruntime OCR 引擎(默认)
|
||||
|
||||
流程:pypdfium2 渲染每页为 PIL Image → rapidocr 识别 → 拼接。
|
||||
类级懒加载单例(_engine/_unavailable 为类属性,跨实例共享),
|
||||
依赖缺失时降级返回空文本。
|
||||
"""
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(io.BytesIO(content))
|
||||
parts: list[str] = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
text_layer = "\n".join(parts).strip()
|
||||
_engine: Any | None = None
|
||||
_unavailable: bool = False
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
if self._unavailable:
|
||||
return ""
|
||||
|
||||
# 1. 懒加载 OCR 引擎(类级单例,跨实例共享)
|
||||
if self._engine is None:
|
||||
try:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
RapidocrOcrEngine._engine = RapidOCR()
|
||||
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
|
||||
except Exception:
|
||||
RapidocrOcrEngine._unavailable = True
|
||||
logger.warning(
|
||||
"OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
# 2. 渲染并识别
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
result, _ = self._engine(pil_image)
|
||||
if result:
|
||||
lines = [
|
||||
item[1]
|
||||
for item in result
|
||||
if item and len(item) >= 2 and item[1]
|
||||
]
|
||||
if lines:
|
||||
page_texts.append("\n".join(lines))
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 备选 OCR 插件:tesseract(lazy import,依赖缺失时不可用)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TesseractOcrEngine:
|
||||
"""pytesseract + pdfium OCR(备选)
|
||||
|
||||
需要系统安装 tesseract 二进制与语言包。
|
||||
类级 _unavailable 单例,跨实例共享。
|
||||
"""
|
||||
|
||||
_unavailable: bool = False
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
if self._unavailable:
|
||||
return ""
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
import pytesseract # type: ignore[import-untyped]
|
||||
from PIL import Image # type: ignore[import-untyped]
|
||||
except Exception:
|
||||
TesseractOcrEngine._unavailable = True
|
||||
logger.warning("tesseract 依赖不可用,降级返回空文本", exc_info=True)
|
||||
return ""
|
||||
|
||||
try:
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
text = pytesseract.image_to_string(pil_image, lang="chi_sim+eng")
|
||||
if text:
|
||||
page_texts.append(text)
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"tesseract OCR 失败,降级返回空文本", error=str(exc), exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
class NoopOcrEngine:
|
||||
"""关闭 OCR 的占位插件(strategy=none 时使用)"""
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 插件注册表
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_PDF_PLUGINS: dict[str, type[PdfTextExtractor]] = {
|
||||
"pypdf": PypdfTextExtractor,
|
||||
"pdfplumber": PdfplumberTextExtractor,
|
||||
}
|
||||
|
||||
_DOCX_PLUGINS: dict[str, type[DocxParser]] = {
|
||||
"python_docx": PythonDocxParser,
|
||||
}
|
||||
|
||||
_OCR_PLUGINS: dict[str, type[OcrEngine]] = {
|
||||
"rapidocr": RapidocrOcrEngine,
|
||||
"tesseract": TesseractOcrEngine,
|
||||
"none": NoopOcrEngine,
|
||||
}
|
||||
|
||||
|
||||
def list_pdf_plugins() -> list[str]:
|
||||
return list(_PDF_PLUGINS.keys())
|
||||
|
||||
|
||||
def list_docx_plugins() -> list[str]:
|
||||
return list(_DOCX_PLUGINS.keys())
|
||||
|
||||
|
||||
def list_ocr_plugins() -> list[str]:
|
||||
return list(_OCR_PLUGINS.keys())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
# 进程级插件单例缓存
|
||||
_pdf_plugin_cache: dict[str, PdfTextExtractor] = {}
|
||||
_docx_plugin_cache: dict[str, DocxParser] = {}
|
||||
_ocr_plugin_cache: dict[str, OcrEngine] = {}
|
||||
|
||||
|
||||
def _get_pdf_plugin() -> PdfTextExtractor:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.pdf.plugin or "pypdf"
|
||||
if name in _pdf_plugin_cache:
|
||||
return _pdf_plugin_cache[name]
|
||||
cls = _PDF_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 PDF 插件,降级到 pypdf", requested=name)
|
||||
cls = PypdfTextExtractor
|
||||
name = "pypdf"
|
||||
instance = cls()
|
||||
_pdf_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def _get_docx_plugin() -> DocxParser:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.docx.plugin or "python_docx"
|
||||
if name in _docx_plugin_cache:
|
||||
return _docx_plugin_cache[name]
|
||||
cls = _DOCX_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 DOCX 插件,降级到 python_docx", requested=name)
|
||||
cls = PythonDocxParser
|
||||
name = "python_docx"
|
||||
instance = cls()
|
||||
_docx_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def _get_ocr_plugin() -> OcrEngine:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.ocr.plugin or "rapidocr"
|
||||
if name in _ocr_plugin_cache:
|
||||
return _ocr_plugin_cache[name]
|
||||
cls = _OCR_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 OCR 插件,降级到 rapidocr", requested=name)
|
||||
cls = RapidocrOcrEngine
|
||||
name = "rapidocr"
|
||||
instance = cls()
|
||||
_ocr_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def invalidate_parser_plugin_cache(kind: str | None = None) -> None:
|
||||
"""清除插件缓存(runtime_settings 更新后调用)"""
|
||||
if kind is None or kind == "pdf":
|
||||
_pdf_plugin_cache.clear()
|
||||
if kind is None or kind == "docx":
|
||||
_docx_plugin_cache.clear()
|
||||
if kind is None or kind == "ocr":
|
||||
_ocr_plugin_cache.clear()
|
||||
# 重置 RapidocrOcrEngine 类级标记,允许重新初始化
|
||||
RapidocrOcrEngine._engine = None
|
||||
RapidocrOcrEngine._unavailable = False
|
||||
TesseractOcrEngine._unavailable = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 顶层解析函数
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _parse_pdf(content: bytes) -> str:
|
||||
"""PDF 解析:优先文本层插件;文本层为空时降级 OCR 插件"""
|
||||
rt = get_runtime_settings()
|
||||
pdf_plugin = _get_pdf_plugin()
|
||||
try:
|
||||
text_layer = pdf_plugin.extract_text(content)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"PDF 文本层提取失败,尝试 OCR 降级",
|
||||
plugin=rt.parsers.pdf.plugin,
|
||||
error=str(exc),
|
||||
exc_info=True,
|
||||
)
|
||||
text_layer = ""
|
||||
|
||||
# 文本层非空:直接返回
|
||||
if text_layer:
|
||||
return text_layer
|
||||
|
||||
# 文本层为空 → 尝试 OCR 降级
|
||||
# 文本层为空 → OCR 降级
|
||||
if not settings.pdf_ocr_enabled:
|
||||
return ""
|
||||
|
||||
ocr_text = _ocr_pdf(
|
||||
content, max_pages=settings.pdf_ocr_max_pages, dpi=settings.pdf_ocr_dpi
|
||||
ocr_plugin = _get_ocr_plugin()
|
||||
return ocr_plugin.ocr_pdf(
|
||||
content,
|
||||
max_pages=rt.parsers.ocr.params.get("max_pages", settings.pdf_ocr_max_pages),
|
||||
dpi=rt.parsers.ocr.params.get("dpi", settings.pdf_ocr_dpi),
|
||||
)
|
||||
return ocr_text
|
||||
|
||||
|
||||
def _ocr_pdf(content: bytes, max_pages: int, dpi: int) -> str:
|
||||
"""对扫描件 PDF 跑 OCR:渲染每页 → 识别 → 拼接
|
||||
|
||||
返回空字符串的场景:依赖未安装 / 渲染或识别异常 / 无识别结果。
|
||||
任何异常仅告警不抛出,由上游按"无法提取文本"处理。
|
||||
"""
|
||||
global _ocr_engine, _ocr_unavailable
|
||||
|
||||
if _ocr_unavailable:
|
||||
return ""
|
||||
|
||||
# 1. 懒加载 OCR 引擎
|
||||
if _ocr_engine is None:
|
||||
try:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
_ocr_engine = RapidOCR()
|
||||
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
|
||||
except Exception:
|
||||
_ocr_unavailable = True
|
||||
logger.warning("OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True)
|
||||
return ""
|
||||
|
||||
# 2. 渲染并识别
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
result, _ = _ocr_engine(pil_image)
|
||||
if result:
|
||||
# result: [[box, text, score], ...],按行拼接
|
||||
lines = [
|
||||
item[1] for item in result if item and len(item) >= 2 and item[1]
|
||||
]
|
||||
if lines:
|
||||
page_texts.append("\n".join(lines))
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning("PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True)
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_docx(content: bytes) -> str:
|
||||
"""DOCX 段落文本拼接(不含表格/页眉页脚)"""
|
||||
from docx import Document # type: ignore[import-untyped]
|
||||
|
||||
document = Document(io.BytesIO(content))
|
||||
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
|
||||
return "\n".join(parts).strip()
|
||||
"""DOCX 解析"""
|
||||
return _get_docx_plugin().extract_text(content)
|
||||
|
||||
|
||||
# 扩展名 → 解析函数映射(启动时构建,避免每次请求重复构造)
|
||||
# 扩展名 → 解析函数映射
|
||||
_PARSERS: dict[str, Callable[[bytes], str]] = {
|
||||
".txt": _parse_text,
|
||||
".md": _parse_text,
|
||||
|
||||
+13
-37
@@ -6,10 +6,12 @@
|
||||
内存注册表为主(记录 status/created_at/updated_at/result/error),
|
||||
Redis 为持久镜像(key: ingest_task:{task_id}),每次状态迁移同步写入;
|
||||
Redis 不可用或写入失败仅记录 warning,不影响任务执行。
|
||||
|
||||
文本去重委托给 app.core.dedup.get_dedup_strategy(),按 runtime_settings.dedup
|
||||
选择策略(none/sha256/simhash)。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
@@ -18,6 +20,7 @@ from typing import Any
|
||||
import structlog
|
||||
|
||||
from app.config import Settings
|
||||
from app.core.dedup import DEDUP_KEY_PREFIX, get_dedup_strategy
|
||||
from app.core.ingestion import Ingester, IngestionError
|
||||
from app.models.document import DocumentInput
|
||||
from app.services.redis import RedisCache
|
||||
@@ -26,8 +29,6 @@ logger = structlog.get_logger()
|
||||
|
||||
# Redis 任务状态 key 前缀
|
||||
REDIS_KEY_PREFIX = "ingest_task:"
|
||||
# Redis 文本去重 key 前缀(value 为已入库文档的 IngestionResult JSON)
|
||||
DEDUP_KEY_PREFIX = "dedup:sha256:"
|
||||
|
||||
|
||||
class IngestTaskStatus(StrEnum):
|
||||
@@ -71,17 +72,17 @@ class IngestTaskManager:
|
||||
async def submit(self, doc: DocumentInput) -> str:
|
||||
"""登记入库任务并后台执行,立即返回 task_id
|
||||
|
||||
文本去重:基于 doc.text 的 sha256 在 Redis 中查重;命中则直接复用旧
|
||||
IngestionResult(仅置 deduplicated=True),不重跑流水线;未命中走原
|
||||
异步入库流程,完成后写入去重记录供后续命中复用。Redis 不可用时跳过
|
||||
去重,按原流程执行,不影响主流程。
|
||||
文本去重:按 runtime_settings.dedup 选择策略(none/sha256/simhash);
|
||||
命中则直接复用旧 IngestionResult(仅置 deduplicated=True),不重跑流水线;
|
||||
未命中走原异步入库流程,完成后写入去重记录供后续命中复用。
|
||||
Redis 不可用或策略=none 时跳过,按原流程执行。
|
||||
"""
|
||||
task_id = uuid.uuid4().hex
|
||||
now = _utc_now_iso()
|
||||
text_hash = hashlib.sha256(doc.text.encode("utf-8")).hexdigest()
|
||||
dedup = get_dedup_strategy(self._redis)
|
||||
|
||||
# 1. 去重命中:直接置 done,复用旧结果,不调 _run
|
||||
dedup_record = await self._lookup_dedup(text_hash)
|
||||
dedup_record = await dedup.lookup(doc.text)
|
||||
if dedup_record is not None:
|
||||
result_dict = dict(dedup_record)
|
||||
result_dict["deduplicated"] = True
|
||||
@@ -111,7 +112,7 @@ class IngestTaskManager:
|
||||
"error": None,
|
||||
}
|
||||
self._schedule_mirror(task_id)
|
||||
background = asyncio.create_task(self._run(task_id, doc, text_hash))
|
||||
background = asyncio.create_task(self._run(task_id, doc, dedup))
|
||||
self._background_tasks.add(background)
|
||||
background.add_done_callback(self._background_tasks.discard)
|
||||
logger.info("入库任务已登记", task_id=task_id, title=doc.title)
|
||||
@@ -151,7 +152,7 @@ class IngestTaskManager:
|
||||
raise TimeoutError(f"入库任务 {task_id} 在 {timeout}s 内未进入终态")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def _run(self, task_id: str, doc: DocumentInput, text_hash: str) -> None:
|
||||
async def _run(self, task_id: str, doc: DocumentInput, dedup: Any) -> None:
|
||||
"""后台执行入库:并发限流 + 阶段状态推进 + 结果/错误落账 + 去重记录写入"""
|
||||
async with self._semaphore:
|
||||
try:
|
||||
@@ -185,34 +186,9 @@ class IngestTaskManager:
|
||||
result=result_dict,
|
||||
)
|
||||
self._schedule_mirror(task_id)
|
||||
await self._record_dedup(text_hash, result_dict)
|
||||
await dedup.record(doc.text, result_dict)
|
||||
logger.info("入库任务完成", task_id=task_id)
|
||||
|
||||
async def _lookup_dedup(self, text_hash: str) -> dict[str, Any] | None:
|
||||
"""查询文本去重记录;Redis 不可用或异常时降级为未命中"""
|
||||
if self._redis is None:
|
||||
return None
|
||||
try:
|
||||
return await self._redis.get_json(f"{DEDUP_KEY_PREFIX}{text_hash}")
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"去重记录查询失败,降级为未命中", text_hash=text_hash, exc_info=True
|
||||
)
|
||||
return None
|
||||
|
||||
async def _record_dedup(self, text_hash: str, result_dict: dict[str, Any]) -> None:
|
||||
"""写入文本去重记录(含完整 IngestionResult),供后续命中复用;失败仅告警"""
|
||||
if self._redis is None:
|
||||
return
|
||||
try:
|
||||
await self._redis.set_json(
|
||||
f"{DEDUP_KEY_PREFIX}{text_hash}",
|
||||
result_dict,
|
||||
ttl=self._settings.ingest_task_ttl_done,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("去重记录写入失败", text_hash=text_hash, exc_info=True)
|
||||
|
||||
def _finish_failed(self, task_id: str, error: dict[str, Any]) -> None:
|
||||
"""将任务置为 failed 并记录错误信息"""
|
||||
self._tasks[task_id].update(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""query 解析与分类路由模块
|
||||
|
||||
分层 RAG 在线侧第一步:用 Ollama 小模型将用户 query 解析为结构化 JSON
|
||||
分层 RAG 在线侧第一步:用 LLM(Ollama 或 OpenAI 兼容服务,由
|
||||
runtime_settings.models.query 决定)将用户 query 解析为结构化 JSON
|
||||
(命中类目+置信度、rewrite 后 query、关键词),再由纯函数做路由决策:
|
||||
- 高置信且命中类目数 <= 上限 → 按类目过滤检索
|
||||
- 低置信 / 解析失败 / 命中类目过多 → 全库兜底(不丢召回)
|
||||
@@ -16,8 +17,8 @@ import structlog
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
from app.config import settings
|
||||
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory, load_taxonomy
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
from app.services.redis import RedisCache, get_cache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
@@ -106,17 +107,22 @@ def decide_route(parsed: ParsedQuery, threshold: float, max_categories: int) ->
|
||||
|
||||
|
||||
class QueryParser:
|
||||
"""query 解析器:调用 Ollama 小模型将 query 解析为结构化 JSON"""
|
||||
"""query 解析器:调用 LLM 将 query 解析为结构化 JSON"""
|
||||
|
||||
def __init__(
|
||||
self, ollama: OllamaClient, taxonomy: list[TaxonomyCategory], cache: RedisCache | None = None
|
||||
self,
|
||||
ollama: LLMClient | None = None,
|
||||
taxonomy: list[TaxonomyCategory] | None = None,
|
||||
cache: RedisCache | None = None,
|
||||
) -> None:
|
||||
self.ollama = ollama
|
||||
self.taxonomy = taxonomy
|
||||
# 默认按 runtime_settings.models.query 选择 LLM 实现;
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("query")
|
||||
self.taxonomy = taxonomy if taxonomy is not None else load_taxonomy()
|
||||
# 解析结果缓存,缺省用全局单例;RedisCache 全操作容错,缓存不可用时退化为无缓存行为
|
||||
self.cache = cache if cache is not None else get_cache()
|
||||
# 可作为路由命中类目的名字集合(uncategorized 不可作为路由命中类目)
|
||||
self._routable_names = {c.name for c in taxonomy if c.name != UNCATEGORIZED}
|
||||
self._routable_names = {c.name for c in self.taxonomy if c.name != UNCATEGORIZED}
|
||||
|
||||
async def parse(self, query: str) -> ParsedQuery:
|
||||
"""调用 LLM 将 query 解析为结构化结果
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""检索结果 AI 总结
|
||||
|
||||
对检索返回的 chunk 命中结果,调用 Ollama 本地模型生成一段针对用户 query 的总结回答。
|
||||
对检索返回的 chunk 命中结果,调用 LLM(Ollama 或 OpenAI 兼容服务,由
|
||||
runtime_settings.models.query 决定)生成一段针对用户 query 的总结回答。
|
||||
仅基于检索结果内容,不编造未提及的信息。
|
||||
"""
|
||||
|
||||
@@ -8,7 +9,7 @@ import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.models.search import SearchHit
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
@@ -16,8 +17,10 @@ logger = structlog.get_logger()
|
||||
class ResultSummarizer:
|
||||
"""检索结果总结器"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
def __init__(self, ollama: LLMClient | None = None) -> None:
|
||||
# 默认按 runtime_settings.models.query 选择 LLM 实现;
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("query")
|
||||
|
||||
async def summarize(self, query: str, hits: list[SearchHit]) -> str:
|
||||
"""对检索结果生成针对 query 的总结
|
||||
|
||||
@@ -21,7 +21,7 @@ from app.core.result_summarizer import ResultSummarizer
|
||||
from app.core.sparse import SparseEncoder
|
||||
from app.models.knowledge import load_taxonomy
|
||||
from app.models.search import ExtractedInfo, SearchHit, SearchRequest, SearchResponse
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import create_llm_client
|
||||
from app.services.qdrant import (
|
||||
COLLECTION_CHUNKS,
|
||||
COLLECTION_L1,
|
||||
@@ -52,7 +52,7 @@ class Retriever:
|
||||
) -> None:
|
||||
self.qdrant = qdrant or QdrantService()
|
||||
self.query_parser = query_parser or QueryParser(
|
||||
ollama=OllamaClient(),
|
||||
ollama=create_llm_client("query"),
|
||||
taxonomy=load_taxonomy(settings.taxonomy_path),
|
||||
)
|
||||
self.embedding = embedding or create_embedding_service()
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""运行时可调配置(runtime settings)
|
||||
|
||||
与 `app.config.settings`(启动时从环境变量加载、不可变)互补:本模块管理运行时
|
||||
可通过管理后台动态调整的子集,持久化到 JSON 文件,启动时加载覆盖到内存单例。
|
||||
|
||||
设计要点:
|
||||
- RuntimeSettings 只包含「可运行时调整」的字段(模型/解析插件/去重等),
|
||||
不包含敏感或启动期固定的字段(端口、数据库连接等)。
|
||||
- 持久化路径默认 `data/runtime_settings.json`,可由 env `RUNTIME_SETTINGS_PATH` 覆盖。
|
||||
- 加载失败/文件缺失时回退到默认值,不阻塞启动。
|
||||
- 写入采用「先临时文件后 rename」原子替换,避免半写损坏。
|
||||
- 全模块只通过 `get_runtime_settings()` 访问单例,避免直接读 JSON。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from typing import Any, Literal
|
||||
|
||||
import structlog
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 默认持久化路径(相对工作目录);可由 env 覆盖
|
||||
DEFAULT_RUNTIME_SETTINGS_PATH = "./data/runtime_settings.json"
|
||||
|
||||
_lock = RLock()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 模型配置(文档总结 / 查询 / 分类 各一份独立配置)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class LlmProviderConfig(BaseModel):
|
||||
"""单个用途的 LLM 提供方配置"""
|
||||
|
||||
provider: Literal["ollama", "openai_compatible"] = Field(
|
||||
default="ollama", description="提供方:ollama 走 Ollama HTTP;openai_compatible 走 OpenAI 兼容 chat/completions"
|
||||
)
|
||||
base_url: str = Field(default="", description="服务地址;空则 ollama 用 settings.ollama_base_url,openai_compatible 用 settings.openai_base_url")
|
||||
api_key: str = Field(default="", description="API Key(仅 openai_compatible 需要;ollama 忽略)")
|
||||
model: str = Field(default="", description="模型名;空则 ollama 用 settings.ollama_model,openai_compatible 用 settings.embedding_model 同级(如 gpt-4o-mini)")
|
||||
timeout: float = Field(default=120.0, description="请求超时秒数")
|
||||
temperature: float = Field(default=0.3, description="采样温度(0~2)")
|
||||
|
||||
|
||||
class ModelSettings(BaseModel):
|
||||
"""模型相关运行时配置:三种用途独立配置"""
|
||||
|
||||
summarize: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档三级总结")
|
||||
query: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="检索链路 query 解析与结果总结")
|
||||
classify: LlmProviderConfig = Field(default_factory=LlmProviderConfig, description="文档分类判定")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 解析插件配置
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class PluginConfig(BaseModel):
|
||||
"""单个解析插件的选择与参数"""
|
||||
|
||||
plugin: str = Field(default="", description="插件名;空则用默认")
|
||||
params: dict[str, Any] = Field(default_factory=dict, description="插件参数(透传给插件实现)")
|
||||
|
||||
|
||||
class ParserSettings(BaseModel):
|
||||
"""解析插件运行时配置"""
|
||||
|
||||
ocr: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="rapidocr"), description="OCR 插件")
|
||||
pdf: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="pypdf"), description="PDF 文本层提取插件")
|
||||
docx: PluginConfig = Field(default_factory=lambda: PluginConfig(plugin="python_docx"), description="DOCX 解析插件")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 去重策略配置
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class DedupSettings(BaseModel):
|
||||
"""文本去重策略运行时配置"""
|
||||
|
||||
strategy: Literal["none", "sha256", "simhash"] = Field(
|
||||
default="sha256", description="去重策略:none 关闭;sha256 精确匹配;simhash 近似匹配"
|
||||
)
|
||||
simhash_threshold: int = Field(default=3, description="simhash 海明距离阈值(仅 strategy=simhash 生效,0~64)")
|
||||
ttl_seconds: int = Field(default=86400, description="去重记录 Redis 保留秒数")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 顶层 RuntimeSettings
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
class RuntimeSettings(BaseModel):
|
||||
"""运行时可调配置顶层模型"""
|
||||
|
||||
models: ModelSettings = Field(default_factory=ModelSettings)
|
||||
parsers: ParserSettings = Field(default_factory=ParserSettings)
|
||||
dedup: DedupSettings = Field(default_factory=DedupSettings)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 单例 + 持久化
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_runtime_settings: RuntimeSettings | None = None
|
||||
|
||||
|
||||
def _resolve_path() -> Path:
|
||||
"""解析持久化文件路径:env RUNTIME_SETTINGS_PATH > settings 自定义 > 默认"""
|
||||
path_str = os.environ.get("RUNTIME_SETTINGS_PATH", "") or getattr(settings, "runtime_settings_path", "") or DEFAULT_RUNTIME_SETTINGS_PATH
|
||||
return Path(path_str).expanduser().resolve()
|
||||
|
||||
|
||||
def _default_with_env_fallback() -> RuntimeSettings:
|
||||
"""构造默认 RuntimeSettings,并把启动 env 中已有的模型相关字段填充进去
|
||||
|
||||
这样首次启动(无持久化文件)时,UI 显示的不是空字符串而是 env 当前值。
|
||||
"""
|
||||
cfg = RuntimeSettings()
|
||||
# 模型默认值沿用 env
|
||||
cfg.models.summarize.base_url = settings.ollama_base_url
|
||||
cfg.models.summarize.model = settings.ollama_model
|
||||
cfg.models.query.base_url = settings.ollama_base_url
|
||||
cfg.models.query.model = settings.ollama_model
|
||||
cfg.models.classify.base_url = settings.ollama_base_url
|
||||
cfg.models.classify.model = settings.ollama_model
|
||||
# 若 env 提供了 OpenAI key/url,预填到 openai_compatible 字段方便切换
|
||||
if settings.openai_api_key:
|
||||
for usage in ("summarize", "query", "classify"):
|
||||
getattr(cfg.models, usage).api_key = settings.openai_api_key
|
||||
getattr(cfg.models, usage).base_url = settings.openai_base_url if settings.openai_base_url else getattr(cfg.models, usage).base_url
|
||||
return cfg
|
||||
|
||||
|
||||
def load_runtime_settings() -> RuntimeSettings:
|
||||
"""从磁盘加载 RuntimeSettings;文件缺失或损坏时回退到默认值(带 env 兜底)"""
|
||||
path = _resolve_path()
|
||||
try:
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return RuntimeSettings.model_validate(data)
|
||||
except Exception:
|
||||
logger.warning("RuntimeSettings 加载失败,回退默认值", path=str(path), exc_info=True)
|
||||
return _default_with_env_fallback()
|
||||
|
||||
|
||||
def save_runtime_settings(cfg: RuntimeSettings) -> None:
|
||||
"""原子写入 RuntimeSettings 到磁盘(先临时文件后 rename)"""
|
||||
path = _resolve_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = cfg.model_dump(mode="json")
|
||||
# 写到同目录临时文件再 rename,避免半写损坏
|
||||
fd, tmp_path = tempfile.mkstemp(prefix=".runtime_settings.", suffix=".tmp", dir=str(path.parent))
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp_path, path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def get_runtime_settings() -> RuntimeSettings:
|
||||
"""获取 RuntimeSettings 单例(首次调用时加载)"""
|
||||
global _runtime_settings
|
||||
with _lock:
|
||||
if _runtime_settings is None:
|
||||
_runtime_settings = load_runtime_settings()
|
||||
return _runtime_settings
|
||||
|
||||
|
||||
def update_runtime_settings(patch: dict[str, Any]) -> RuntimeSettings:
|
||||
"""以 patch 字典更新 RuntimeSettings 并持久化
|
||||
|
||||
支持部分更新(顶层键可缺失,缺省保留原值)。例如:
|
||||
update_runtime_settings({"models": {"summarize": {"model": "qwen2.5:3b"}}})
|
||||
"""
|
||||
with _lock:
|
||||
current = get_runtime_settings()
|
||||
merged = current.model_dump(mode="json")
|
||||
_deep_merge(merged, patch)
|
||||
new_cfg = RuntimeSettings.model_validate(merged)
|
||||
save_runtime_settings(new_cfg)
|
||||
# 替换单例后立即返回,新值对所有后续读取生效
|
||||
global _runtime_settings
|
||||
_runtime_settings = new_cfg
|
||||
logger.info("RuntimeSettings 已更新并持久化", path=str(_resolve_path()))
|
||||
return new_cfg
|
||||
|
||||
|
||||
def _deep_merge(target: dict[str, Any], patch: dict[str, Any]) -> None:
|
||||
"""递归把 patch 合并到 target(同 key 字典则递归,否则覆盖)"""
|
||||
for k, v in patch.items():
|
||||
if k in target and isinstance(target[k], dict) and isinstance(v, dict):
|
||||
_deep_merge(target[k], v)
|
||||
else:
|
||||
target[k] = v
|
||||
|
||||
|
||||
def reload_runtime_settings() -> RuntimeSettings:
|
||||
"""强制从磁盘重新加载(管理后台触发)"""
|
||||
with _lock:
|
||||
global _runtime_settings
|
||||
_runtime_settings = load_runtime_settings()
|
||||
return _runtime_settings
|
||||
|
||||
|
||||
def reset_runtime_settings() -> RuntimeSettings:
|
||||
"""重置为默认值并持久化(管理后台触发)"""
|
||||
with _lock:
|
||||
new_cfg = _default_with_env_fallback()
|
||||
save_runtime_settings(new_cfg)
|
||||
global _runtime_settings
|
||||
_runtime_settings = new_cfg
|
||||
return new_cfg
|
||||
@@ -1,6 +1,7 @@
|
||||
"""文档三级总结模块
|
||||
|
||||
通过 Ollama 本地小模型对文档进行分级总结:
|
||||
通过 LLM(Ollama 或 OpenAI 兼容服务,由 runtime_settings.models.summarize 决定)
|
||||
对文档进行分级总结:
|
||||
- L1: 总结(一句话高度概括)
|
||||
- L2: 大纲(主要章节和关键主题)
|
||||
- L3: 内容大纲(每个章节的详细内容摘要)
|
||||
@@ -12,7 +13,7 @@ import structlog
|
||||
|
||||
from app.core.headings import parse_headings, render_outline
|
||||
from app.models.document import DocumentSummary, SummaryLevel
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.llm import LLMClient, create_llm_client
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
@@ -23,8 +24,10 @@ MIN_TEXT_LENGTH_FOR_L3 = 500
|
||||
class Summarizer:
|
||||
"""文档三级总结器"""
|
||||
|
||||
def __init__(self, ollama: OllamaClient | None = None) -> None:
|
||||
self.ollama = ollama or OllamaClient()
|
||||
def __init__(self, ollama: LLMClient | None = None) -> None:
|
||||
# 默认按 runtime_settings 选择 LLM 实现(Ollama 或 OpenAI 兼容);
|
||||
# 测试可通过 ollama 参数注入替身。
|
||||
self.ollama = ollama or create_llm_client("summarize")
|
||||
|
||||
async def summarize(self, text: str, *, title: str = "") -> DocumentSummary:
|
||||
"""对文档文本进行三级总结
|
||||
|
||||
+50
-4
@@ -5,13 +5,14 @@ from pathlib import Path
|
||||
import structlog
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
|
||||
|
||||
from app.api.response import ApiError, error
|
||||
from app.api.v1.auth import router as auth_router
|
||||
from app.api.v1.document import router as document_router
|
||||
from app.api.v1.knowledge import router as knowledge_router
|
||||
from app.api.v1.search import router as search_router
|
||||
from app.api.v1.settings import router as settings_router
|
||||
from app.config import settings
|
||||
from app.core.auth import ensure_default_admin
|
||||
from app.services.qdrant import QdrantService
|
||||
@@ -53,6 +54,7 @@ app.include_router(auth_router)
|
||||
app.include_router(search_router)
|
||||
app.include_router(document_router)
|
||||
app.include_router(knowledge_router)
|
||||
app.include_router(settings_router)
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
@@ -84,10 +86,54 @@ async def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
_ADMIN_HTML = Path(__file__).resolve().parent / "static" / "admin.html"
|
||||
_STATIC_DIR = Path(__file__).resolve().parent / "static"
|
||||
_ADMIN_HTML = _STATIC_DIR / "admin.html"
|
||||
_ADMIN_SPA_DIR = _STATIC_DIR / "admin"
|
||||
|
||||
|
||||
def _spa_index() -> Path | None:
|
||||
"""SPA 入口文件路径;不存在返回 None(回退旧 admin.html)"""
|
||||
index = _ADMIN_SPA_DIR / "index.html"
|
||||
return index if index.is_file() else None
|
||||
|
||||
|
||||
@app.get("/admin", include_in_schema=False)
|
||||
async def admin_page() -> FileResponse:
|
||||
"""管理后台单页(单文件静态 HTML,零外部依赖)"""
|
||||
async def admin_page():
|
||||
"""管理后台:优先服务 Vue SPA,回退到旧单文件 admin.html"""
|
||||
spa = _spa_index()
|
||||
if spa is not None:
|
||||
return FileResponse(spa, media_type="text/html")
|
||||
return FileResponse(_ADMIN_HTML, media_type="text/html")
|
||||
|
||||
|
||||
@app.get("/admin/", include_in_schema=False)
|
||||
async def admin_page_trailing_slash():
|
||||
"""带斜杠的 /admin/ 重定向到 /admin"""
|
||||
return RedirectResponse(url="/admin", status_code=307)
|
||||
|
||||
|
||||
@app.get("/admin/{rest:path}", include_in_schema=False)
|
||||
async def admin_spa(rest: str):
|
||||
"""SPA 静态资源与客户端路由兜底
|
||||
|
||||
- 真实文件(assets/*.js, *.css 等)→ 直接返回
|
||||
- 其余路径(/overview, /documents 等客户端路由)→ 返回 index.html
|
||||
- SPA 目录不存在时 → 404(旧 admin.html 无子路由需求)
|
||||
"""
|
||||
spa_dir = _ADMIN_SPA_DIR
|
||||
if not spa_dir.is_dir():
|
||||
return JSONResponse(
|
||||
status_code=404, content={"code": 1002, "message": "Not found"}
|
||||
)
|
||||
|
||||
# 安全校验:防止路径越界
|
||||
file_path = (spa_dir / rest).resolve()
|
||||
try:
|
||||
file_path.relative_to(spa_dir.resolve())
|
||||
except ValueError:
|
||||
return FileResponse(spa_dir / "index.html", media_type="text/html")
|
||||
|
||||
if file_path.is_file():
|
||||
return FileResponse(file_path)
|
||||
# SPA 客户端路由兜底
|
||||
return FileResponse(spa_dir / "index.html", media_type="text/html")
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
"""LLM 客户端抽象层
|
||||
|
||||
提供统一的 `generate(prompt, json_mode)` 接口,底层实现可切换:
|
||||
- OllamaLLMClient:走 Ollama HTTP /api/generate(本地或远程 Ollama 服务)
|
||||
- OpenAICompatibleLLMClient:走 OpenAI 兼容 /v1/chat/completions(OpenAI / DeepSeek / 智谱 / Qwen API 等)
|
||||
|
||||
工厂 `create_llm_client(purpose)` 根据 runtime_settings 选择实现与参数。
|
||||
所有客户端共享同一接口,业务侧无需感知底层协议差异。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal, Protocol, runtime_checkable
|
||||
|
||||
import httpx
|
||||
import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.core.runtime_settings import LlmProviderConfig, get_runtime_settings
|
||||
from app.services.ollama import OllamaClient
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
LlmPurpose = Literal["summarize", "query", "classify"]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class LLMClient(Protocol):
|
||||
"""LLM 客户端统一接口"""
|
||||
|
||||
async def generate(self, prompt: str, json_mode: bool = False) -> str:
|
||||
"""生成文本
|
||||
|
||||
Args:
|
||||
prompt: 输入提示词
|
||||
json_mode: True 时约束输出为 JSON(不支持时降级为普通生成)
|
||||
|
||||
Returns:
|
||||
生成的文本内容
|
||||
"""
|
||||
...
|
||||
|
||||
async def is_available(self) -> bool:
|
||||
"""检查服务是否可用"""
|
||||
...
|
||||
|
||||
|
||||
class OllamaLLMClient:
|
||||
"""Ollama HTTP API 客户端(包装现有 OllamaClient,便于统一接口)"""
|
||||
|
||||
def __init__(self, base_url: str, model: str, timeout: float = 120.0) -> None:
|
||||
self._inner = OllamaClient(base_url=base_url, model=model, timeout=timeout)
|
||||
|
||||
async def generate(self, prompt: str, json_mode: bool = False) -> str:
|
||||
return await self._inner.generate(prompt, json_mode=json_mode)
|
||||
|
||||
async def is_available(self) -> bool:
|
||||
return await self._inner.is_available()
|
||||
|
||||
|
||||
class OpenAICompatibleLLMClient:
|
||||
"""OpenAI 兼容 chat/completions 客户端
|
||||
|
||||
适用于 OpenAI 官方 API、DeepSeek、智谱 ChatGLM、Qwen DashScope、Moonshot 等
|
||||
所有兼容 OpenAI /v1/chat/completions 协议的服务。
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str, model: str, timeout: float = 120.0, temperature: float = 0.3) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self.timeout = timeout
|
||||
self.temperature = temperature
|
||||
|
||||
async def generate(self, prompt: str, json_mode: bool = False) -> str:
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
payload: dict = {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": self.temperature,
|
||||
"stream": False,
|
||||
}
|
||||
if json_mode:
|
||||
payload["response_format"] = {"type": "json_object"}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
resp = await client.post(url, json=payload, headers=headers)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# OpenAI 标准响应结构:choices[0].message.content
|
||||
choices = data.get("choices") or []
|
||||
if not choices:
|
||||
logger.warning("OpenAI 兼容响应无 choices", model=self.model, raw_keys=list(data.keys()))
|
||||
return ""
|
||||
message = choices[0].get("message") or {}
|
||||
content = message.get("content") or ""
|
||||
logger.debug("OpenAI 兼容生成完成", model=self.model, output_length=len(content))
|
||||
return content
|
||||
|
||||
async def is_available(self) -> bool:
|
||||
"""简单探测:调 /models 列表接口(多数 OpenAI 兼容服务支持)"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/models",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
)
|
||||
return resp.status_code == 200
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
# 进程级客户端缓存(避免每请求新建 httpx client)
|
||||
_client_cache: dict[str, LLMClient] = {}
|
||||
|
||||
|
||||
def _resolve_provider_config(purpose: LlmPurpose) -> LlmProviderConfig:
|
||||
"""从 runtime_settings 取指定用途的 LLM 配置"""
|
||||
rt = get_runtime_settings()
|
||||
return getattr(rt.models, purpose)
|
||||
|
||||
|
||||
def _build_client(purpose: LlmPurpose) -> LLMClient:
|
||||
"""按 runtime_settings 构造 LLM 客户端"""
|
||||
cfg = _resolve_provider_config(purpose)
|
||||
|
||||
if cfg.provider == "ollama":
|
||||
base_url = cfg.base_url or settings.ollama_base_url
|
||||
model = cfg.model or settings.ollama_model
|
||||
return OllamaLLMClient(base_url=base_url, model=model, timeout=cfg.timeout)
|
||||
|
||||
if cfg.provider == "openai_compatible":
|
||||
base_url = cfg.base_url or settings.openai_base_url
|
||||
api_key = cfg.api_key or settings.openai_api_key
|
||||
# openai_compatible 默认模型:若 cfg.model 为空,退化到 OpenAI 通用 chat 模型
|
||||
model = cfg.model or "gpt-4o-mini"
|
||||
if not api_key:
|
||||
logger.warning("OpenAI 兼容 provider 缺少 api_key,调用大概率会失败", purpose=purpose)
|
||||
return OpenAICompatibleLLMClient(
|
||||
base_url=base_url, api_key=api_key, model=model, timeout=cfg.timeout, temperature=cfg.temperature
|
||||
)
|
||||
|
||||
raise ValueError(f"未知 LLM provider: {cfg.provider}")
|
||||
|
||||
|
||||
def create_llm_client(purpose: LlmPurpose, *, use_cache: bool = True) -> LLMClient:
|
||||
"""创建指定用途的 LLM 客户端
|
||||
|
||||
Args:
|
||||
purpose: 用途(summarize / query / classify)
|
||||
use_cache: True 时复用进程级客户端单例(默认);False 每次新建(测试用)
|
||||
|
||||
Returns:
|
||||
LLMClient 实例
|
||||
"""
|
||||
if not use_cache:
|
||||
return _build_client(purpose)
|
||||
|
||||
cache_key = f"{purpose}"
|
||||
if cache_key not in _client_cache:
|
||||
_client_cache[cache_key] = _build_client(purpose)
|
||||
return _client_cache[cache_key]
|
||||
|
||||
|
||||
def invalidate_llm_client_cache(purpose: LlmPurpose | None = None) -> None:
|
||||
"""清除客户端缓存(runtime_settings 更新后调用,确保后续读取新配置)"""
|
||||
if purpose is None:
|
||||
_client_cache.clear()
|
||||
else:
|
||||
_client_cache.pop(purpose, None)
|
||||
+22
-5
@@ -10,6 +10,8 @@ services:
|
||||
- QDRANT_PORT=6333
|
||||
- REDIS_URL=redis://redis:6379/0
|
||||
- OLLAMA_BASE_URL=http://ollama:11434
|
||||
# 针对 16 线程 / 61GB 内存的 NAS 调优:放宽入库并发
|
||||
- INGEST_MAX_CONCURRENCY=${INGEST_MAX_CONCURRENCY:-4}
|
||||
env_file:
|
||||
- .env
|
||||
depends_on:
|
||||
@@ -21,6 +23,7 @@ services:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- ${NAS_DATA_DIR:-./data}/logs:/app/logs
|
||||
- ${NAS_DATA_DIR:-./data}/uploads:/app/uploads
|
||||
networks:
|
||||
- qmdsearch
|
||||
|
||||
@@ -65,11 +68,25 @@ services:
|
||||
- "${OLLAMA_PORT:-11434}:11434"
|
||||
volumes:
|
||||
- ${NAS_DATA_DIR:-./data}/ollama:/root/.ollama
|
||||
# 首次启动后需手动拉取模型:
|
||||
# docker exec qmdsearch-ollama ollama pull qwen2.5:1.5b
|
||||
# 或取消下方 entrypoint 注释以自动拉取(需等待下载完成)
|
||||
# entrypoint: /bin/bash
|
||||
# command: -c "ollama serve & sleep 5 && ollama pull qwen2.5:1.5b && wait"
|
||||
environment:
|
||||
# 针对 Ryzen 9 7940HS(16 线程)的 CPU 推理调优:
|
||||
# 并行推理任务数、常驻模型数、单请求线程上限、KV 缓存量化以省内存
|
||||
- OLLAMA_NUM_PARALLEL=4
|
||||
- OLLAMA_MAX_LOADED_MODELS=2
|
||||
- OLLAMA_NUM_THREADS=16
|
||||
- OLLAMA_KV_CACHE_TYPE=q8_0
|
||||
# 首次启动自动拉取所需模型(qwen2.5:1.5b 总结 + bge-m3 嵌入),
|
||||
# 下载完成后转交常驻 ollama serve。已存在时仅做健康检查。
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- -c
|
||||
- |
|
||||
ollama serve &
|
||||
SERVE_PID=$$!
|
||||
sleep 6
|
||||
ollama pull qwen2.5:1.5b
|
||||
ollama pull bge-m3
|
||||
wait $$SERVE_PID
|
||||
networks:
|
||||
- qmdsearch
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
.DS_Store
|
||||
.vite
|
||||
@@ -0,0 +1,12 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>QMDSearch 知识库管理后台</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"jsx": "preserve",
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
},
|
||||
"checkJs": false
|
||||
},
|
||||
"include": ["src/**/*.js", "src/**/*.vue"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Generated
+1871
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "qmdsearch-frontend",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ant-design/icons-vue": "^7.0.1",
|
||||
"ant-design-vue": "^4.2.6",
|
||||
"axios": "^1.7.9",
|
||||
"dayjs": "^1.11.13",
|
||||
"pinia": "^2.3.0",
|
||||
"vue": "^3.5.13",
|
||||
"vue-router": "^4.5.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@vitejs/plugin-vue": "^5.2.1",
|
||||
"vite": "^5.4.11"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
<script setup>
|
||||
import { RouterView } from 'vue-router'
|
||||
import { ConfigProvider, theme as antdTheme } from 'ant-design-vue'
|
||||
|
||||
const themeConfig = {
|
||||
algorithm: antdTheme.defaultAlgorithm,
|
||||
token: {
|
||||
colorPrimary: '#1677ff',
|
||||
colorInfo: '#1677ff',
|
||||
colorSuccess: '#52c41a',
|
||||
colorWarning: '#faad14',
|
||||
colorError: '#ff4d4f',
|
||||
borderRadius: 6,
|
||||
fontSize: 14,
|
||||
fontFamily:
|
||||
"-apple-system, BlinkMacSystemFont, 'PingFang SC', 'Microsoft YaHei', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif"
|
||||
},
|
||||
components: {
|
||||
Layout: {
|
||||
headerBg: '#ffffff',
|
||||
headerPadding: '0 16px',
|
||||
headerHeight: 52,
|
||||
siderBg: '#001529'
|
||||
},
|
||||
Menu: {
|
||||
darkItemBg: '#001529',
|
||||
darkSubMenuItemBg: '#000c17'
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<ConfigProvider :theme="themeConfig" :locale="undefined">
|
||||
<RouterView />
|
||||
</ConfigProvider>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
/* 全局根容器无额外样式,由各布局/页面自行处理 */
|
||||
</style>
|
||||
@@ -0,0 +1,19 @@
|
||||
import http from './client'
|
||||
|
||||
/**
|
||||
* 用户名密码登录
|
||||
* @param {string} username
|
||||
* @param {string} password
|
||||
* @returns {Promise<{access_token:string, expires_in:number, user:{username:string, role:string, created_at:string}}>}
|
||||
*/
|
||||
export function login(username, password) {
|
||||
return http.post('/api/v1/auth/login', { username, password })
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前登录用户信息
|
||||
* @returns {Promise<{username:string, role:string, created_at:string}>}
|
||||
*/
|
||||
export function me() {
|
||||
return http.get('/api/v1/auth/me')
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import axios from 'axios'
|
||||
import { message } from 'ant-design-vue'
|
||||
|
||||
const TOKEN_STORAGE_KEY = 'qmd_token'
|
||||
|
||||
/** 认证相关错误码:触发清 token + 跳登录 */
|
||||
const AUTH_ERROR_CODES = new Set([1003, 1005])
|
||||
|
||||
const httpClient = axios.create({
|
||||
// 不设 baseURL,使用相对路径,由 vite proxy / nginx 转发
|
||||
timeout: 60000,
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
})
|
||||
|
||||
// 请求拦截器:注入 Bearer token
|
||||
httpClient.interceptors.request.use((config) => {
|
||||
const token = localStorage.getItem(TOKEN_STORAGE_KEY) || ''
|
||||
if (token) {
|
||||
config.headers = config.headers || {}
|
||||
config.headers.Authorization = `Bearer ${token}`
|
||||
}
|
||||
return config
|
||||
})
|
||||
|
||||
let unauthorizedHandler = null
|
||||
|
||||
/**
|
||||
* 注册 401 / 认证错误处理回调(由 router/store 注入,避免循环依赖)
|
||||
* @param {() => void} handler
|
||||
*/
|
||||
export function setUnauthorizedHandler(handler) {
|
||||
unauthorizedHandler = handler
|
||||
}
|
||||
|
||||
function triggerUnauthorized() {
|
||||
localStorage.removeItem(TOKEN_STORAGE_KEY)
|
||||
if (typeof unauthorizedHandler === 'function') {
|
||||
unauthorizedHandler()
|
||||
}
|
||||
}
|
||||
|
||||
// 响应拦截器:统一处理 code !== 0 与 401
|
||||
httpClient.interceptors.response.use(
|
||||
(response) => {
|
||||
const body = response.data
|
||||
if (body && typeof body === 'object' && 'code' in body) {
|
||||
if (body.code === 0) {
|
||||
return body.data
|
||||
}
|
||||
// 业务错误
|
||||
if (AUTH_ERROR_CODES.has(body.code)) {
|
||||
triggerUnauthorized()
|
||||
}
|
||||
const err = new Error(body.message || '请求失败')
|
||||
err.code = body.code
|
||||
err.message = body.message || '请求失败'
|
||||
return Promise.reject(err)
|
||||
}
|
||||
// 非标准结构,原样返回
|
||||
return body
|
||||
},
|
||||
(error) => {
|
||||
const status = error?.response?.status
|
||||
if (status === 401) {
|
||||
triggerUnauthorized()
|
||||
const err = new Error('未认证或登录已过期,请重新登录')
|
||||
err.code = 1003
|
||||
return Promise.reject(err)
|
||||
}
|
||||
// 后端返回了 body 但 HTTP 错误
|
||||
const body = error?.response?.data
|
||||
if (body && typeof body === 'object' && 'code' in body) {
|
||||
if (AUTH_ERROR_CODES.has(body.code)) {
|
||||
triggerUnauthorized()
|
||||
}
|
||||
const err = new Error(body.message || `请求失败 (HTTP ${status ?? '?'})`)
|
||||
err.code = body.code
|
||||
return Promise.reject(err)
|
||||
}
|
||||
const err = new Error(error?.message || '网络请求失败')
|
||||
err.code = `HTTP_${status ?? 'NETWORK'}`
|
||||
return Promise.reject(err)
|
||||
}
|
||||
)
|
||||
|
||||
/**
|
||||
* 统一发起请求,捕获异常并弹出 antd message
|
||||
* @param {() => Promise<any>} fn
|
||||
* @param {{ silent?: boolean, errorText?: string }} [options]
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
export async function callApi(fn, options = {}) {
|
||||
const { silent = false, errorText = '操作失败' } = options
|
||||
try {
|
||||
return await fn()
|
||||
} catch (err) {
|
||||
const text = err?.message || errorText
|
||||
if (!silent) {
|
||||
message.error(text)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export { TOKEN_STORAGE_KEY }
|
||||
export default httpClient
|
||||
@@ -0,0 +1,62 @@
|
||||
import http from './client'
|
||||
|
||||
/**
|
||||
* 分页列出文档
|
||||
* @param {number} [limit=20]
|
||||
* @param {string|null} [offset=null]
|
||||
* @returns {Promise<{items: Array, next_offset: string|null}>}
|
||||
*/
|
||||
export function list(limit = 20, offset = null) {
|
||||
const params = { limit }
|
||||
if (offset !== null && offset !== undefined && offset !== '') {
|
||||
params.offset = offset
|
||||
}
|
||||
return http.get('/api/v1/documents', { params })
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文档详情
|
||||
* @param {string} docId
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
export function detail(docId) {
|
||||
return http.get(`/api/v1/documents/${encodeURIComponent(docId)}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文档(幂等)
|
||||
* @param {string} docId
|
||||
* @returns {Promise<{doc_id:string, deleted: object, deleted_total:number}>}
|
||||
*/
|
||||
export function remove(docId) {
|
||||
return http.delete(`/api/v1/documents/${encodeURIComponent(docId)}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON 文本入库(异步)
|
||||
* @param {{title:string, source?:string, text:string, metadata?:object}} payload
|
||||
* @returns {Promise<{task_id:string, status:string}>}
|
||||
*/
|
||||
export function ingest(payload) {
|
||||
return http.post('/api/v1/documents', payload)
|
||||
}
|
||||
|
||||
/**
|
||||
* multipart 文件上传入库
|
||||
* @param {FormData} formData
|
||||
* @returns {Promise<{task_id:string, status:string, saved_path?:string}>}
|
||||
*/
|
||||
export function upload(formData) {
|
||||
return http.post('/api/v1/documents/upload', formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询入库任务状态
|
||||
* @param {string} taskId
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
export function taskStatus(taskId) {
|
||||
return http.get(`/api/v1/documents/tasks/${encodeURIComponent(taskId)}`)
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import http from './client'
|
||||
|
||||
/**
|
||||
* 获取知识分类类目集
|
||||
* @returns {Promise<{categories: Array<{name:string, description:string}>, count: number}>}
|
||||
*/
|
||||
export function categories() {
|
||||
return http.get('/api/v1/knowledge/categories')
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取知识库统计(四层点数 + 类目分布 + uncategorized 数)
|
||||
* @returns {Promise<{collections: {doc_l1:number, doc_l2:number, doc_l3:number, chunks:number}, documents_total:number, uncategorized_count:number, categories: Record<string, number>}>}
|
||||
*/
|
||||
export function stats() {
|
||||
return http.get('/api/v1/knowledge/stats')
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import http from './client'
|
||||
|
||||
/**
|
||||
* 分层检索
|
||||
* @param {{query:string, top_k?:number, summarize?:boolean}} payload
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
export function search(payload) {
|
||||
return http.post('/api/v1/search', payload)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import http from './client'
|
||||
|
||||
/**
|
||||
* 获取当前 RuntimeSettings
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
export function get() {
|
||||
return http.get('/api/v1/settings')
|
||||
}
|
||||
|
||||
/**
|
||||
* 部分更新 RuntimeSettings(仅 admin)
|
||||
* @param {{models?:object, parsers?:object, dedup?:object}} payload
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
export function update(payload) {
|
||||
return http.put('/api/v1/settings', payload)
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取可选项 schema
|
||||
* @returns {Promise<{llm_providers:string[], pdf_plugins:string[], docx_plugins:string[], ocr_plugins:string[], dedup_strategies:string[]}>}
|
||||
*/
|
||||
export function schema() {
|
||||
return http.get('/api/v1/settings/schema')
|
||||
}
|
||||
|
||||
/**
|
||||
* 重置为默认值(仅 admin)
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
export function reset() {
|
||||
return http.post('/api/v1/settings/reset')
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { onBeforeUnmount, reactive, ref } from 'vue'
|
||||
import { taskStatus } from '@/api/documents'
|
||||
import {
|
||||
INGEST_POLL_INTERVAL_MS,
|
||||
INGEST_POLL_MAX_ATTEMPTS,
|
||||
INGEST_TERMINAL_STATUS
|
||||
} from '@/constants/ingest'
|
||||
|
||||
/**
|
||||
* 入库任务轮询 composable
|
||||
*
|
||||
* 调用 startPolling(taskId) 启动轮询;到达 done/failed/超时/出错 时自动停止并
|
||||
* 写入 state。组件卸载时自动清理定时器。
|
||||
*
|
||||
* @returns {{
|
||||
* state: { taskId: string|null, task: object|null, status: string|null, error: object|null, isTimeout: boolean, isPolling: boolean },
|
||||
* startPolling: (taskId: string) => void,
|
||||
* stopPolling: () => void
|
||||
* }}
|
||||
*/
|
||||
export function useIngestPolling() {
|
||||
const state = reactive({
|
||||
taskId: null,
|
||||
task: null,
|
||||
status: null,
|
||||
error: null,
|
||||
isTimeout: false,
|
||||
isPolling: false
|
||||
})
|
||||
|
||||
const timerRef = ref(null)
|
||||
let attempts = 0
|
||||
|
||||
function stopPolling() {
|
||||
if (timerRef.value !== null) {
|
||||
clearInterval(timerRef.value)
|
||||
timerRef.value = null
|
||||
}
|
||||
state.isPolling = false
|
||||
}
|
||||
|
||||
function reset() {
|
||||
stopPolling()
|
||||
state.taskId = null
|
||||
state.task = null
|
||||
state.status = null
|
||||
state.error = null
|
||||
state.isTimeout = false
|
||||
state.isPolling = false
|
||||
attempts = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动轮询
|
||||
* @param {string} taskId
|
||||
*/
|
||||
function startPolling(taskId) {
|
||||
reset()
|
||||
state.taskId = taskId
|
||||
state.isPolling = true
|
||||
attempts = 0
|
||||
|
||||
timerRef.value = setInterval(async () => {
|
||||
attempts += 1
|
||||
if (attempts > INGEST_POLL_MAX_ATTEMPTS) {
|
||||
stopPolling()
|
||||
state.isTimeout = true
|
||||
return
|
||||
}
|
||||
try {
|
||||
const task = await taskStatus(taskId)
|
||||
state.task = task
|
||||
state.status = task?.status || null
|
||||
if (INGEST_TERMINAL_STATUS.includes(task?.status)) {
|
||||
stopPolling()
|
||||
}
|
||||
} catch (err) {
|
||||
state.error = err
|
||||
stopPolling()
|
||||
}
|
||||
}, INGEST_POLL_INTERVAL_MS)
|
||||
}
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
stopPolling()
|
||||
})
|
||||
|
||||
return {
|
||||
state,
|
||||
startPolling,
|
||||
stopPolling
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* 入库任务状态映射:后端 status → 中文展示
|
||||
*/
|
||||
export const INGEST_STATUS_TEXT = Object.freeze({
|
||||
pending: '排队中',
|
||||
summarizing: '总结中',
|
||||
classifying: '分类中',
|
||||
embedding: '向量化中',
|
||||
writing: '写入中',
|
||||
done: '完成',
|
||||
failed: '失败'
|
||||
})
|
||||
|
||||
/** 轮询间隔(毫秒) */
|
||||
export const INGEST_POLL_INTERVAL_MS = 2000
|
||||
|
||||
/** 轮询最大次数:150 次 × 2s = 5 分钟超时 */
|
||||
export const INGEST_POLL_MAX_ATTEMPTS = 150
|
||||
|
||||
/** 入库状态徽标颜色映射(antd Badge / Tag 状态) */
|
||||
export const INGEST_STATUS_COLOR = Object.freeze({
|
||||
pending: 'default',
|
||||
summarizing: 'processing',
|
||||
classifying: 'processing',
|
||||
embedding: 'processing',
|
||||
writing: 'processing',
|
||||
done: 'success',
|
||||
failed: 'error'
|
||||
})
|
||||
|
||||
/** 终态集合:到达这些状态后停止轮询 */
|
||||
export const INGEST_TERMINAL_STATUS = Object.freeze(['done', 'failed'])
|
||||
@@ -0,0 +1,232 @@
|
||||
<script setup>
|
||||
import { computed, h, ref } from 'vue'
|
||||
import { useRoute, useRouter, RouterView } from 'vue-router'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { Modal } from 'ant-design-vue'
|
||||
import {
|
||||
DashboardOutlined,
|
||||
FileTextOutlined,
|
||||
UploadOutlined,
|
||||
SearchOutlined,
|
||||
AppstoreOutlined,
|
||||
SettingOutlined,
|
||||
MenuFoldOutlined,
|
||||
MenuUnfoldOutlined,
|
||||
LogoutOutlined,
|
||||
DatabaseOutlined
|
||||
} from '@ant-design/icons-vue'
|
||||
import { useAuthStore } from '@/stores/useAuthStore'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
const authStore = useAuthStore()
|
||||
const { user, displayName } = storeToRefs(authStore)
|
||||
|
||||
const collapsed = ref(false)
|
||||
|
||||
const selectedKeys = computed(() => {
|
||||
return [route.name ? String(route.name) : '']
|
||||
})
|
||||
|
||||
const openKeys = ref(['main'])
|
||||
|
||||
const menuItems = [
|
||||
{ key: 'overview', icon: () => h(DashboardOutlined), label: '概览' },
|
||||
{ key: 'documents', icon: () => h(FileTextOutlined), label: '文档管理' },
|
||||
{ key: 'ingest', icon: () => h(UploadOutlined), label: '文档入库' },
|
||||
{ key: 'search', icon: () => h(SearchOutlined), label: '检索测试台' },
|
||||
{ key: 'categories', icon: () => h(AppstoreOutlined), label: '类目列表' },
|
||||
{ key: 'settings', icon: () => h(SettingOutlined), label: '设置' }
|
||||
]
|
||||
|
||||
function handleMenuClick({ key }) {
|
||||
if (key && key !== route.name) {
|
||||
router.push({ name: key })
|
||||
}
|
||||
}
|
||||
|
||||
function handleLogout() {
|
||||
Modal.confirm({
|
||||
title: '确认登出',
|
||||
content: '确定要退出登录吗?',
|
||||
okText: '登出',
|
||||
cancelText: '取消',
|
||||
onOk() {
|
||||
authStore.logout()
|
||||
router.replace({ name: 'login' })
|
||||
}
|
||||
})
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<a-layout class="main-layout">
|
||||
<a-layout-sider
|
||||
v-model:collapsed="collapsed"
|
||||
collapsible
|
||||
:trigger="null"
|
||||
breakpoint="lg"
|
||||
class="main-layout__sider"
|
||||
>
|
||||
<div class="main-layout__logo">
|
||||
<span class="main-layout__logo-icon">
|
||||
<DatabaseOutlined />
|
||||
</span>
|
||||
<span v-if="!collapsed" class="main-layout__logo-text">QMDSearch</span>
|
||||
</div>
|
||||
<a-menu
|
||||
theme="dark"
|
||||
mode="inline"
|
||||
:selected-keys="selectedKeys"
|
||||
:open-keys="openKeys"
|
||||
:items="menuItems"
|
||||
@click="handleMenuClick"
|
||||
/>
|
||||
</a-layout-sider>
|
||||
|
||||
<a-layout>
|
||||
<a-layout-header class="main-layout__header">
|
||||
<a-button
|
||||
type="text"
|
||||
class="main-layout__collapse"
|
||||
@click="collapsed = !collapsed"
|
||||
>
|
||||
<MenuUnfoldOutlined v-if="collapsed" />
|
||||
<MenuFoldOutlined v-else />
|
||||
</a-button>
|
||||
<div class="main-layout__title">知识库管理后台</div>
|
||||
<div class="main-layout__user">
|
||||
<span class="main-layout__user-name">{{ displayName }}</span>
|
||||
<a-divider type="vertical" />
|
||||
<a-button
|
||||
type="text"
|
||||
size="small"
|
||||
class="main-layout__logout"
|
||||
@click="handleLogout"
|
||||
>
|
||||
<template #icon><LogoutOutlined /></template>
|
||||
登出
|
||||
</a-button>
|
||||
</div>
|
||||
</a-layout-header>
|
||||
|
||||
<a-layout-content class="main-layout__content">
|
||||
<RouterView />
|
||||
</a-layout-content>
|
||||
|
||||
<a-layout-footer class="main-layout__footer">
|
||||
QMDSearch Admin · 当前用户:{{ user?.username || '-' }}
|
||||
</a-layout-footer>
|
||||
</a-layout>
|
||||
</a-layout>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.main-layout {
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
.main-layout__sider {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
height: 100vh;
|
||||
overflow: auto;
|
||||
box-shadow: 2px 0 8px rgba(0, 21, 41, 0.15);
|
||||
}
|
||||
|
||||
.main-layout__logo {
|
||||
height: 52px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 10px;
|
||||
padding: 0 16px;
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
background: linear-gradient(135deg, #001529 0%, #002140 100%);
|
||||
}
|
||||
|
||||
.main-layout__logo-icon {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
border-radius: 6px;
|
||||
background: linear-gradient(135deg, #1677ff 0%, #4096ff 100%);
|
||||
color: #fff;
|
||||
font-size: 16px;
|
||||
box-shadow: 0 2px 6px rgba(22, 119, 255, 0.4);
|
||||
}
|
||||
|
||||
.main-layout__logo-text {
|
||||
color: #fff;
|
||||
font-size: 17px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.5px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.main-layout__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
background: #fff;
|
||||
padding: 0 20px;
|
||||
height: 52px;
|
||||
line-height: 52px;
|
||||
box-shadow: 0 1px 4px rgba(0, 21, 41, 0.08);
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
.main-layout__collapse {
|
||||
flex: 0 0 auto;
|
||||
font-size: 16px;
|
||||
color: #4b5563;
|
||||
}
|
||||
|
||||
.main-layout__collapse:hover {
|
||||
color: #1677ff;
|
||||
}
|
||||
|
||||
.main-layout__title {
|
||||
flex: 1 1 auto;
|
||||
margin-left: 12px;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
color: #1f2937;
|
||||
}
|
||||
|
||||
.main-layout__user {
|
||||
flex: 0 0 auto;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.main-layout__user-name {
|
||||
color: #4b5563;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.main-layout__logout {
|
||||
color: #6b7280;
|
||||
}
|
||||
|
||||
.main-layout__logout:hover {
|
||||
color: #ff4d4f;
|
||||
}
|
||||
|
||||
.main-layout__content {
|
||||
margin: 16px;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.main-layout__footer {
|
||||
text-align: center;
|
||||
color: #6b7280;
|
||||
font-size: 12px;
|
||||
background: transparent;
|
||||
padding: 12px 16px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,30 @@
|
||||
import { createApp } from 'vue'
|
||||
import { createPinia } from 'pinia'
|
||||
import Antd from 'ant-design-vue'
|
||||
import 'ant-design-vue/dist/reset.css'
|
||||
import App from './App.vue'
|
||||
import router from './router'
|
||||
import { setUnauthorizedHandler } from './api/client'
|
||||
import { useAuthStore } from './stores/useAuthStore'
|
||||
import './styles/main.css'
|
||||
|
||||
const app = createApp(App)
|
||||
const pinia = createPinia()
|
||||
|
||||
app.use(pinia)
|
||||
|
||||
// 注册 401 处理:清 store + 跳 /login
|
||||
const authStore = useAuthStore()
|
||||
setUnauthorizedHandler(() => {
|
||||
authStore.clearAuth()
|
||||
if (router.currentRoute.value.name !== 'login') {
|
||||
router.replace({
|
||||
name: 'login',
|
||||
query: { redirect: router.currentRoute.value.fullPath }
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
app.use(router)
|
||||
app.use(Antd)
|
||||
app.mount('#app')
|
||||
@@ -0,0 +1,91 @@
|
||||
import { createRouter, createWebHistory } from 'vue-router'
|
||||
import { useAuthStore } from '@/stores/useAuthStore'
|
||||
|
||||
const routes = [
|
||||
{
|
||||
path: '/login',
|
||||
name: 'login',
|
||||
component: () => import('@/views/Login.vue'),
|
||||
meta: { title: '登录', requiresAuth: false }
|
||||
},
|
||||
{
|
||||
path: '/',
|
||||
component: () => import('@/layouts/MainLayout.vue'),
|
||||
redirect: '/overview',
|
||||
meta: { requiresAuth: true },
|
||||
children: [
|
||||
{
|
||||
path: 'overview',
|
||||
name: 'overview',
|
||||
component: () => import('@/views/Overview.vue'),
|
||||
meta: { title: '概览', requiresAuth: true }
|
||||
},
|
||||
{
|
||||
path: 'documents',
|
||||
name: 'documents',
|
||||
component: () => import('@/views/Documents.vue'),
|
||||
meta: { title: '文档管理', requiresAuth: true }
|
||||
},
|
||||
{
|
||||
path: 'ingest',
|
||||
name: 'ingest',
|
||||
component: () => import('@/views/Ingest.vue'),
|
||||
meta: { title: '文档入库', requiresAuth: true }
|
||||
},
|
||||
{
|
||||
path: 'search',
|
||||
name: 'search',
|
||||
component: () => import('@/views/Search.vue'),
|
||||
meta: { title: '检索测试台', requiresAuth: true }
|
||||
},
|
||||
{
|
||||
path: 'categories',
|
||||
name: 'categories',
|
||||
component: () => import('@/views/Categories.vue'),
|
||||
meta: { title: '类目列表', requiresAuth: true }
|
||||
},
|
||||
{
|
||||
path: 'settings',
|
||||
name: 'settings',
|
||||
component: () => import('@/views/Settings.vue'),
|
||||
meta: { title: '设置', requiresAuth: true }
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
path: '/:pathMatch(.*)*',
|
||||
name: 'not-found',
|
||||
redirect: '/overview'
|
||||
}
|
||||
]
|
||||
|
||||
const router = createRouter({
|
||||
history: createWebHistory('/admin/'),
|
||||
routes,
|
||||
scrollBehavior() {
|
||||
return { top: 0 }
|
||||
}
|
||||
})
|
||||
|
||||
// 全局前置守卫:未登录跳 /login;已登录访问 /login 跳 /overview
|
||||
router.beforeEach((to) => {
|
||||
const authStore = useAuthStore()
|
||||
const title = to.meta?.title
|
||||
if (title) {
|
||||
document.title = `${title} - QMDSearch 知识库后台`
|
||||
} else {
|
||||
document.title = 'QMDSearch 知识库后台'
|
||||
}
|
||||
|
||||
if (to.meta?.requiresAuth && !authStore.isAuthenticated) {
|
||||
return { name: 'login', query: { redirect: to.fullPath } }
|
||||
}
|
||||
|
||||
if (to.name === 'login' && authStore.isAuthenticated) {
|
||||
return { name: 'overview' }
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
export default router
|
||||
@@ -0,0 +1,66 @@
|
||||
import { defineStore } from 'pinia'
|
||||
|
||||
const TOKEN_STORAGE_KEY = 'qmd_token'
|
||||
const USER_STORAGE_KEY = 'qmd_user'
|
||||
|
||||
/**
|
||||
* 从 localStorage 读取用户信息
|
||||
* @returns {{username:string, role:string, created_at?:string} | null}
|
||||
*/
|
||||
function loadUserFromStorage() {
|
||||
try {
|
||||
const raw = localStorage.getItem(USER_STORAGE_KEY)
|
||||
if (!raw) return null
|
||||
const parsed = JSON.parse(raw)
|
||||
if (parsed && typeof parsed === 'object' && parsed.username) {
|
||||
return parsed
|
||||
}
|
||||
return null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export const useAuthStore = defineStore('auth', {
|
||||
state: () => ({
|
||||
token: localStorage.getItem(TOKEN_STORAGE_KEY) || '',
|
||||
user: loadUserFromStorage()
|
||||
}),
|
||||
|
||||
getters: {
|
||||
isAuthenticated: (state) => Boolean(state.token),
|
||||
isAdmin: (state) => state.user?.role === 'admin',
|
||||
displayName: (state) => {
|
||||
if (!state.user) return ''
|
||||
return `${state.user.username} (${state.user.role})`
|
||||
}
|
||||
},
|
||||
|
||||
actions: {
|
||||
/**
|
||||
* 登录成功后保存 token + user
|
||||
* @param {{access_token:string, user:object}} data
|
||||
*/
|
||||
setAuth(data) {
|
||||
this.token = data.access_token
|
||||
this.user = data.user
|
||||
localStorage.setItem(TOKEN_STORAGE_KEY, data.access_token)
|
||||
localStorage.setItem(USER_STORAGE_KEY, JSON.stringify(data.user))
|
||||
},
|
||||
|
||||
/** 清除登录态(登出 / 401) */
|
||||
clearAuth() {
|
||||
this.token = ''
|
||||
this.user = null
|
||||
localStorage.removeItem(TOKEN_STORAGE_KEY)
|
||||
localStorage.removeItem(USER_STORAGE_KEY)
|
||||
},
|
||||
|
||||
/**
|
||||
* 登出
|
||||
*/
|
||||
logout() {
|
||||
this.clearAuth()
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,79 @@
|
||||
html,
|
||||
body,
|
||||
#app {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
#app {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'PingFang SC', 'Microsoft YaHei',
|
||||
'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
|
||||
color: #1f2937;
|
||||
background: #f0f2f5;
|
||||
}
|
||||
|
||||
/* 统一文本工具类 */
|
||||
.text-muted {
|
||||
color: #6b7280;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.text-break {
|
||||
word-break: break-word;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
/* 页面通用 section 容器 */
|
||||
.page-section {
|
||||
background: #fff;
|
||||
border-radius: 8px;
|
||||
padding: 20px 24px;
|
||||
box-shadow: 0 1px 2px rgba(0, 21, 41, 0.04);
|
||||
}
|
||||
|
||||
/* flex-gap:横排卡片/标签 */
|
||||
.flex-gap {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.toolbar {
|
||||
margin: 12px 0;
|
||||
}
|
||||
|
||||
/* 统一页面标题样式 */
|
||||
.page-title {
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
margin: 0;
|
||||
color: #1f2937;
|
||||
}
|
||||
|
||||
/* 统一卡片标题样式 */
|
||||
.section-subtitle {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #374151;
|
||||
margin: 16px 0 12px;
|
||||
}
|
||||
|
||||
/* 滚动条美化 */
|
||||
::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: rgba(0, 0, 0, 0.2);
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb:hover {
|
||||
background: rgba(0, 0, 0, 0.35);
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* 截断文本,超过最大长度追加省略号
|
||||
* @param {string} text
|
||||
* @param {number} [maxLen=80]
|
||||
* @returns {string}
|
||||
*/
|
||||
export function truncate(text, maxLen = 80) {
|
||||
if (!text) return ''
|
||||
const s = String(text)
|
||||
return s.length > maxLen ? `${s.slice(0, maxLen)}…` : s
|
||||
}
|
||||
|
||||
/**
|
||||
* 安全拼接类名
|
||||
* @param {...(string | false | null | undefined)} args
|
||||
* @returns {string}
|
||||
*/
|
||||
export function classnames(...args) {
|
||||
return args.filter(Boolean).join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* 防抖
|
||||
* @param {Function} fn
|
||||
* @param {number} [wait=300]
|
||||
* @returns {Function}
|
||||
*/
|
||||
export function debounce(fn, wait = 300) {
|
||||
let timer = null
|
||||
return function debounced(...args) {
|
||||
if (timer) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
timer = setTimeout(() => {
|
||||
fn.apply(this, args)
|
||||
}, wait)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
<script setup>
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { message } from 'ant-design-vue'
|
||||
import { categories as fetchCategories } from '@/api/knowledge'
|
||||
|
||||
const isLoading = ref(false)
|
||||
const categoriesData = ref([])
|
||||
|
||||
const columns = [
|
||||
{ title: '名称', dataIndex: 'name', key: 'name' },
|
||||
{ title: '描述', dataIndex: 'description', key: 'description' }
|
||||
]
|
||||
|
||||
async function loadCategories() {
|
||||
isLoading.value = true
|
||||
try {
|
||||
const data = await fetchCategories()
|
||||
categoriesData.value = data.categories || []
|
||||
} catch (err) {
|
||||
message.error(err?.message || '加载类目列表失败')
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
loadCategories()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="categories page-section">
|
||||
<div class="categories__header">
|
||||
<h2 class="categories__title">类目列表</h2>
|
||||
<a-button :loading="isLoading" @click="loadCategories">刷新</a-button>
|
||||
</div>
|
||||
|
||||
<a-table
|
||||
:columns="columns"
|
||||
:data-source="categoriesData"
|
||||
:pagination="false"
|
||||
:loading="isLoading"
|
||||
row-key="name"
|
||||
size="small"
|
||||
>
|
||||
<template #bodyCell="{ column, record }">
|
||||
<template v-if="column.key === 'name'">
|
||||
<a-tag color="blue">{{ record.name }}</a-tag>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'description'">
|
||||
<span>{{ record.description || '-' }}</span>
|
||||
</template>
|
||||
</template>
|
||||
</a-table>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.categories__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.categories__title {
|
||||
font-size: 16px;
|
||||
margin: 0;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,299 @@
|
||||
<script setup>
|
||||
import { onMounted, reactive, ref } from 'vue'
|
||||
import { message, Modal } from 'ant-design-vue'
|
||||
import {
|
||||
list as fetchDocuments,
|
||||
detail as fetchDocumentDetail,
|
||||
remove as deleteDocument
|
||||
} from '@/api/documents'
|
||||
import { truncate } from '@/utils/format'
|
||||
|
||||
const isLoading = ref(false)
|
||||
const isLoadingDetail = ref(false)
|
||||
const isDeleting = ref(false)
|
||||
|
||||
const documents = ref([])
|
||||
const nextOffset = ref(null)
|
||||
|
||||
const detailVisible = ref(false)
|
||||
const detailData = ref(null)
|
||||
|
||||
const statusText = reactive({
|
||||
text: ''
|
||||
})
|
||||
|
||||
function updateStatusText() {
|
||||
statusText.text =
|
||||
nextOffset.value === null ? '已加载全部' : '还有更多,可继续加载'
|
||||
}
|
||||
|
||||
async function loadDocuments(reset = false) {
|
||||
isLoading.value = true
|
||||
try {
|
||||
const offset = reset ? null : nextOffset.value
|
||||
const data = await fetchDocuments(20, offset)
|
||||
if (reset) {
|
||||
documents.value = data.items || []
|
||||
} else {
|
||||
documents.value = documents.value.concat(data.items || [])
|
||||
}
|
||||
nextOffset.value = data.next_offset ?? null
|
||||
updateStatusText()
|
||||
} catch (err) {
|
||||
message.error(err?.message || '加载文档列表失败')
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function handleLoadMore() {
|
||||
await loadDocuments(false)
|
||||
}
|
||||
|
||||
function handleViewDetail(docId) {
|
||||
detailVisible.value = true
|
||||
detailData.value = null
|
||||
loadDocDetail(docId)
|
||||
}
|
||||
|
||||
async function loadDocDetail(docId) {
|
||||
isLoadingDetail.value = true
|
||||
try {
|
||||
detailData.value = await fetchDocumentDetail(docId)
|
||||
} catch (err) {
|
||||
message.error(err?.message || '加载文档详情失败')
|
||||
} finally {
|
||||
isLoadingDetail.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function handleDelete(doc) {
|
||||
const title = doc.title || doc.doc_id
|
||||
Modal.confirm({
|
||||
title: '确认删除',
|
||||
content: `确定删除文档「${title}」(${doc.doc_id}) 吗?该操作将删除四层集合中的全部数据,不可恢复。`,
|
||||
okText: '删除',
|
||||
okType: 'danger',
|
||||
cancelText: '取消',
|
||||
async onOk() {
|
||||
isDeleting.value = true
|
||||
try {
|
||||
const data = await deleteDocument(doc.doc_id)
|
||||
message.success(`已删除 ${data.deleted_total ?? 0} 条数据`)
|
||||
detailVisible.value = false
|
||||
await loadDocuments(true)
|
||||
} catch (err) {
|
||||
message.error(err?.message || '删除文档失败')
|
||||
} finally {
|
||||
isDeleting.value = false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function handleCloseDetail() {
|
||||
detailVisible.value = false
|
||||
detailData.value = null
|
||||
}
|
||||
|
||||
const columns = [
|
||||
{ title: '标题', dataIndex: 'title', key: 'title', ellipsis: true },
|
||||
{ title: '类目', dataIndex: 'category', key: 'category', width: 140 },
|
||||
{ title: '标签', dataIndex: 'tags', key: 'tags', width: 220 },
|
||||
{ title: 'L1 摘要', dataIndex: 'summary', key: 'summary', ellipsis: true },
|
||||
{ title: '操作', key: 'action', width: 160, fixed: 'right' }
|
||||
]
|
||||
|
||||
function getDocTitle(record) {
|
||||
return record?.title || '(无标题)'
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
loadDocuments(true)
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="documents page-section">
|
||||
<div class="documents__header">
|
||||
<h2 class="documents__title">文档管理</h2>
|
||||
</div>
|
||||
|
||||
<a-table
|
||||
:columns="columns"
|
||||
:data-source="documents"
|
||||
:pagination="false"
|
||||
:loading="isLoading"
|
||||
row-key="doc_id"
|
||||
size="small"
|
||||
:scroll="{ x: 900 }"
|
||||
>
|
||||
<template #bodyCell="{ column, record }">
|
||||
<template v-if="column.key === 'title'">
|
||||
<span :title="record.title">{{ getDocTitle(record) }}</span>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'category'">
|
||||
<a-tag v-if="record.category" color="blue">{{ record.category }}</a-tag>
|
||||
<span v-else class="text-muted">-</span>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'tags'">
|
||||
<template v-if="record.tags && record.tags.length">
|
||||
<a-tag v-for="tag in record.tags" :key="tag">{{ tag }}</a-tag>
|
||||
</template>
|
||||
<span v-else class="text-muted">-</span>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'summary'">
|
||||
<span :title="record.summary">{{ truncate(record.summary, 80) }}</span>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'action'">
|
||||
<a-space>
|
||||
<a-button type="link" size="small" @click="handleViewDetail(record.doc_id)">
|
||||
详情
|
||||
</a-button>
|
||||
<a-button
|
||||
type="link"
|
||||
size="small"
|
||||
danger
|
||||
:loading="isDeleting"
|
||||
@click="handleDelete(record)"
|
||||
>
|
||||
删除
|
||||
</a-button>
|
||||
</a-space>
|
||||
</template>
|
||||
</template>
|
||||
</a-table>
|
||||
|
||||
<div class="toolbar documents__toolbar">
|
||||
<a-button
|
||||
:loading="isLoading"
|
||||
:disabled="nextOffset === null"
|
||||
@click="handleLoadMore"
|
||||
>
|
||||
加载更多
|
||||
</a-button>
|
||||
<span class="text-muted" style="margin-left: 12px">{{ statusText.text }}</span>
|
||||
</div>
|
||||
|
||||
<a-drawer
|
||||
:open="detailVisible"
|
||||
title="文档详情"
|
||||
placement="right"
|
||||
width="640"
|
||||
:destroy-on-close="true"
|
||||
@close="handleCloseDetail"
|
||||
>
|
||||
<a-spin :spinning="isLoadingDetail">
|
||||
<div v-if="detailData">
|
||||
<a-descriptions :column="1" size="small" bordered>
|
||||
<a-descriptions-item label="doc_id">
|
||||
{{ detailData.l1?.doc_id || '-' }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="标题">
|
||||
{{ detailData.l1?.title || '-' }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="类目">
|
||||
{{ detailData.l1?.category || '-' }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="标签">
|
||||
<template v-if="detailData.l1?.tags && detailData.l1.tags.length">
|
||||
<a-tag v-for="tag in detailData.l1.tags" :key="tag">{{ tag }}</a-tag>
|
||||
</template>
|
||||
<span v-else class="text-muted">-</span>
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="chunks_count">
|
||||
{{ detailData.chunks_count ?? 0 }}
|
||||
</a-descriptions-item>
|
||||
</a-descriptions>
|
||||
|
||||
<h3 class="documents__section-title">L1 全文</h3>
|
||||
<pre class="documents__pre">{{ detailData.l1?.text || '' }}</pre>
|
||||
|
||||
<h3 class="documents__section-title">
|
||||
L2 节点({{ (detailData.l2_nodes || []).length }})
|
||||
</h3>
|
||||
<div v-if="(detailData.l2_nodes || []).length === 0" class="text-muted">无</div>
|
||||
<div
|
||||
v-for="(node, idx) in detailData.l2_nodes || []"
|
||||
:key="`l2-${idx}`"
|
||||
class="documents__node"
|
||||
>
|
||||
<div class="documents__node-path">{{ node.section_path || '' }}</div>
|
||||
<div class="documents__node-text text-break">{{ node.text || '' }}</div>
|
||||
</div>
|
||||
|
||||
<h3 class="documents__section-title">
|
||||
L3 节点({{ (detailData.l3_nodes || []).length }})
|
||||
</h3>
|
||||
<div v-if="(detailData.l3_nodes || []).length === 0" class="text-muted">无</div>
|
||||
<div
|
||||
v-for="(node, idx) in detailData.l3_nodes || []"
|
||||
:key="`l3-${idx}`"
|
||||
class="documents__node"
|
||||
>
|
||||
<div class="documents__node-path">{{ node.section_path || '' }}</div>
|
||||
<div class="documents__node-text text-break">{{ node.text || '' }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div v-else-if="!isLoadingDetail" class="text-muted">暂无数据</div>
|
||||
</a-spin>
|
||||
</a-drawer>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.documents__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.documents__title {
|
||||
font-size: 16px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.documents__toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.documents__section-title {
|
||||
font-size: 14px;
|
||||
margin: 16px 0 8px;
|
||||
color: #374151;
|
||||
}
|
||||
|
||||
.documents__pre {
|
||||
background: #f9fafb;
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 4px;
|
||||
padding: 8px;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-size: 12px;
|
||||
max-height: 260px;
|
||||
overflow: auto;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.documents__node {
|
||||
border-left: 3px solid #93c5fd;
|
||||
padding: 6px 10px;
|
||||
margin-bottom: 6px;
|
||||
background: #f8fafc;
|
||||
border-radius: 0 4px 4px 0;
|
||||
}
|
||||
|
||||
.documents__node-path {
|
||||
font-size: 12px;
|
||||
color: #6b7280;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.documents__node-text {
|
||||
font-size: 13px;
|
||||
color: #1f2937;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,352 @@
|
||||
<script setup>
|
||||
import { computed, reactive, ref } from 'vue'
|
||||
import { message } from 'ant-design-vue'
|
||||
import { ingest as ingestDocument, upload as uploadDocument } from '@/api/documents'
|
||||
import { useIngestPolling } from '@/composables/useIngestPolling'
|
||||
import {
|
||||
INGEST_STATUS_TEXT,
|
||||
INGEST_STATUS_COLOR
|
||||
} from '@/constants/ingest'
|
||||
|
||||
const { state: pollState, startPolling, stopPolling } = useIngestPolling()
|
||||
|
||||
const activeTab = ref('text')
|
||||
|
||||
const isSubmittingText = ref(false)
|
||||
const isSubmittingFile = ref(false)
|
||||
|
||||
const textFormRef = ref(null)
|
||||
const textForm = reactive({
|
||||
title: '',
|
||||
source: '',
|
||||
text: ''
|
||||
})
|
||||
|
||||
const textRules = {
|
||||
title: [{ required: true, message: '请输入标题', trigger: 'blur' }],
|
||||
text: [{ required: true, message: '请输入正文', trigger: 'blur' }]
|
||||
}
|
||||
|
||||
const fileForm = reactive({
|
||||
title: '',
|
||||
source: ''
|
||||
})
|
||||
const fileList = ref([])
|
||||
const rawFile = ref(null)
|
||||
|
||||
const ACCEPTED_EXTENSIONS = '.txt,.md,.html,.htm,.pdf,.docx'
|
||||
|
||||
const statusBadgeColor = computed(() => {
|
||||
return INGEST_STATUS_COLOR[pollState.status] || 'default'
|
||||
})
|
||||
|
||||
const statusBadgeText = computed(() => {
|
||||
return INGEST_STATUS_TEXT[pollState.status] || pollState.status || '-'
|
||||
})
|
||||
|
||||
const isTerminal = computed(() =>
|
||||
['done', 'failed'].includes(pollState.status)
|
||||
)
|
||||
|
||||
const resultData = computed(() => pollState.task?.result || null)
|
||||
const errorData = computed(() => pollState.task?.error || pollState.error || null)
|
||||
const summaryData = computed(() => resultData.value?.summary || {})
|
||||
const partialSummary = computed(() => errorData.value?.partial_summary || null)
|
||||
|
||||
function handleFileChange(file) {
|
||||
// a-upload before-upload 返回 false 表示不自动上传;保存原始 File 供 FormData 使用
|
||||
rawFile.value = file
|
||||
fileList.value = [file]
|
||||
return false
|
||||
}
|
||||
|
||||
function handleFileRemove() {
|
||||
rawFile.value = null
|
||||
fileList.value = []
|
||||
}
|
||||
|
||||
async function handleSubmitText() {
|
||||
try {
|
||||
await textFormRef.value.validate()
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
isSubmittingText.value = true
|
||||
try {
|
||||
const payload = {
|
||||
title: textForm.title,
|
||||
text: textForm.text
|
||||
}
|
||||
if (textForm.source) {
|
||||
payload.source = textForm.source
|
||||
}
|
||||
const data = await ingestDocument(payload)
|
||||
message.success('任务已提交')
|
||||
startPolling(data.task_id)
|
||||
} catch (err) {
|
||||
message.error(err?.message || '提交入库失败')
|
||||
} finally {
|
||||
isSubmittingText.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSubmitFile() {
|
||||
if (!rawFile.value) {
|
||||
message.warning('请选择文件')
|
||||
return
|
||||
}
|
||||
isSubmittingFile.value = true
|
||||
try {
|
||||
const formData = new FormData()
|
||||
formData.append('file', rawFile.value)
|
||||
if (fileForm.title) {
|
||||
formData.append('title', fileForm.title)
|
||||
}
|
||||
if (fileForm.source) {
|
||||
formData.append('source', fileForm.source)
|
||||
}
|
||||
const data = await uploadDocument(formData)
|
||||
message.success('任务已提交')
|
||||
startPolling(data.task_id)
|
||||
} catch (err) {
|
||||
message.error(err?.message || '上传入库失败')
|
||||
} finally {
|
||||
isSubmittingFile.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function handleCancelPolling() {
|
||||
stopPolling()
|
||||
message.info('已停止轮询')
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="ingest page-section">
|
||||
<div class="ingest__header">
|
||||
<h2 class="ingest__title">文档入库</h2>
|
||||
</div>
|
||||
|
||||
<a-tabs v-model:activeKey="activeTab">
|
||||
<a-tab-pane key="text" tab="文本入库">
|
||||
<a-form
|
||||
ref="textFormRef"
|
||||
:model="textForm"
|
||||
:rules="textRules"
|
||||
layout="vertical"
|
||||
>
|
||||
<a-form-item label="标题" name="title">
|
||||
<a-input v-model:value="textForm.title" placeholder="请输入标题" allow-clear />
|
||||
</a-form-item>
|
||||
<a-form-item label="来源" name="source">
|
||||
<a-input
|
||||
v-model:value="textForm.source"
|
||||
placeholder="例如:manual / web / file"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item label="正文" name="text">
|
||||
<a-textarea
|
||||
v-model:value="textForm.text"
|
||||
placeholder="请输入文档正文"
|
||||
:auto-size="{ minRows: 8, maxRows: 18 }"
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item>
|
||||
<a-button
|
||||
type="primary"
|
||||
:loading="isSubmittingText || pollState.isPolling"
|
||||
@click="handleSubmitText"
|
||||
>
|
||||
提交入库
|
||||
</a-button>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</a-tab-pane>
|
||||
|
||||
<a-tab-pane key="file" tab="文件上传">
|
||||
<a-form
|
||||
:model="fileForm"
|
||||
layout="vertical"
|
||||
>
|
||||
<a-form-item label="选择文件">
|
||||
<a-upload
|
||||
:file-list="fileList"
|
||||
:accept="ACCEPTED_EXTENSIONS"
|
||||
:max-count="1"
|
||||
:before-upload="handleFileChange"
|
||||
@remove="handleFileRemove"
|
||||
>
|
||||
<a-button :disabled="fileList.length >= 1">选择文件</a-button>
|
||||
</a-upload>
|
||||
<div class="text-muted" style="margin-top: 4px">
|
||||
支持 .txt/.md/.html/.htm/.pdf/.docx
|
||||
</div>
|
||||
</a-form-item>
|
||||
<a-form-item label="标题(可选,默认取文件名)" name="title">
|
||||
<a-input
|
||||
v-model:value="fileForm.title"
|
||||
placeholder="留空则使用文件名去扩展"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item label="来源(可选,默认 file:原文件名)" name="source">
|
||||
<a-input
|
||||
v-model:value="fileForm.source"
|
||||
placeholder="例如:manual / web"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item>
|
||||
<a-button
|
||||
type="primary"
|
||||
:loading="isSubmittingFile || pollState.isPolling"
|
||||
@click="handleSubmitFile"
|
||||
>
|
||||
上传入库
|
||||
</a-button>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</a-tab-pane>
|
||||
</a-tabs>
|
||||
|
||||
<div v-if="pollState.taskId" class="ingest__result">
|
||||
<div class="ingest__result-header">
|
||||
<span>任务已提交:{{ pollState.taskId }}</span>
|
||||
<a-tag :color="statusBadgeColor">{{ statusBadgeText }}</a-tag>
|
||||
<a-button
|
||||
v-if="pollState.isPolling"
|
||||
type="link"
|
||||
size="small"
|
||||
@click="handleCancelPolling"
|
||||
>
|
||||
停止轮询
|
||||
</a-button>
|
||||
</div>
|
||||
|
||||
<a-alert
|
||||
v-if="pollState.isTimeout"
|
||||
class="ingest__alert"
|
||||
type="warning"
|
||||
show-icon
|
||||
:message="`任务仍在进行,可稍后凭 task_id 查询:${pollState.taskId}`"
|
||||
/>
|
||||
|
||||
<a-alert
|
||||
v-if="pollState.error && !pollState.task"
|
||||
class="ingest__alert"
|
||||
type="error"
|
||||
show-icon
|
||||
:message="pollState.error?.message || '轮询失败'"
|
||||
/>
|
||||
|
||||
<div v-if="isTerminal && pollState.status === 'done' && resultData" class="ingest__done">
|
||||
<a-descriptions :column="1" size="small" bordered>
|
||||
<a-descriptions-item label="document_id">
|
||||
{{ resultData.document_id || '-' }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="类目">
|
||||
{{ resultData.category || '-' }}(置信度 {{ resultData.category_confidence ?? '-' }})
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="标签">
|
||||
<template v-if="resultData.tags && resultData.tags.length">
|
||||
<a-tag v-for="tag in resultData.tags" :key="tag">{{ tag }}</a-tag>
|
||||
</template>
|
||||
<span v-else class="text-muted">-</span>
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="总结层级">
|
||||
{{ summaryData.level ?? '-' }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="写入集合">
|
||||
{{ resultData.collection || '-' }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="chunks_count">
|
||||
{{ resultData.chunks_count ?? 0 }}
|
||||
</a-descriptions-item>
|
||||
</a-descriptions>
|
||||
<h3 class="ingest__section-title">L1 总结</h3>
|
||||
<pre class="ingest__pre">{{ summaryData.l1_summary || '' }}</pre>
|
||||
</div>
|
||||
|
||||
<div v-if="isTerminal && pollState.status === 'failed'" class="ingest__failed">
|
||||
<a-descriptions :column="1" size="small" bordered>
|
||||
<a-descriptions-item label="失败阶段">
|
||||
{{ errorData?.stage || '-' }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="错误信息">
|
||||
<span class="text-break">{{ errorData?.message || '-' }}</span>
|
||||
</a-descriptions-item>
|
||||
</a-descriptions>
|
||||
<template v-if="partialSummary && partialSummary.l1_summary">
|
||||
<a-alert
|
||||
class="ingest__alert"
|
||||
type="info"
|
||||
show-icon
|
||||
message="已产出总结保留:任务失败前已生成 L1 摘要"
|
||||
/>
|
||||
<h3 class="ingest__section-title">L1 总结</h3>
|
||||
<pre class="ingest__pre">{{ partialSummary.l1_summary }}</pre>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.ingest__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.ingest__title {
|
||||
font-size: 16px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.ingest__result {
|
||||
margin-top: 16px;
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 6px;
|
||||
background: #fafafa;
|
||||
padding: 12px 16px;
|
||||
}
|
||||
|
||||
.ingest__result-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
font-size: 13px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.ingest__alert {
|
||||
margin: 8px 0;
|
||||
}
|
||||
|
||||
.ingest__done,
|
||||
.ingest__failed {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.ingest__section-title {
|
||||
font-size: 14px;
|
||||
margin: 12px 0 8px;
|
||||
color: #374151;
|
||||
}
|
||||
|
||||
.ingest__pre {
|
||||
background: #f9fafb;
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 4px;
|
||||
padding: 8px;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-size: 12px;
|
||||
max-height: 260px;
|
||||
overflow: auto;
|
||||
margin: 0;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,199 @@
|
||||
<script setup>
|
||||
import { reactive, ref } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { message } from 'ant-design-vue'
|
||||
import { UserOutlined, LockOutlined, DatabaseOutlined } from '@ant-design/icons-vue'
|
||||
import { login as loginApi } from '@/api/auth'
|
||||
import { useAuthStore } from '@/stores/useAuthStore'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
const authStore = useAuthStore()
|
||||
|
||||
const isLoading = ref(false)
|
||||
const formRef = ref(null)
|
||||
|
||||
const formState = reactive({
|
||||
username: '',
|
||||
password: ''
|
||||
})
|
||||
|
||||
const rules = {
|
||||
username: [{ required: true, message: '请输入用户名', trigger: 'blur' }],
|
||||
password: [{ required: true, message: '请输入密码', trigger: 'blur' }]
|
||||
}
|
||||
|
||||
async function handleSubmit() {
|
||||
try {
|
||||
await formRef.value.validate()
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
isLoading.value = true
|
||||
try {
|
||||
const data = await loginApi(formState.username, formState.password)
|
||||
authStore.setAuth(data)
|
||||
message.success('登录成功')
|
||||
const redirect = typeof route.query.redirect === 'string' ? route.query.redirect : '/overview'
|
||||
router.replace(redirect)
|
||||
} catch (err) {
|
||||
message.error(err?.message || '登录失败')
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="login-page">
|
||||
<div class="login-page__bg-deco login-page__bg-deco--1" />
|
||||
<div class="login-page__bg-deco login-page__bg-deco--2" />
|
||||
<div class="login-page__box">
|
||||
<div class="login-page__brand">
|
||||
<div class="login-page__logo">
|
||||
<DatabaseOutlined />
|
||||
</div>
|
||||
<div class="login-page__brand-text">
|
||||
<div class="login-page__brand-name">QMDSearch</div>
|
||||
<div class="login-page__brand-sub">知识库管理后台</div>
|
||||
</div>
|
||||
</div>
|
||||
<a-form
|
||||
ref="formRef"
|
||||
:model="formState"
|
||||
:rules="rules"
|
||||
layout="vertical"
|
||||
@finish="handleSubmit"
|
||||
>
|
||||
<a-form-item label="用户名" name="username">
|
||||
<a-input
|
||||
v-model:value="formState.username"
|
||||
placeholder="请输入用户名"
|
||||
autocomplete="username"
|
||||
allow-clear
|
||||
size="large"
|
||||
>
|
||||
<template #prefix><UserOutlined /></template>
|
||||
</a-input>
|
||||
</a-form-item>
|
||||
<a-form-item label="密码" name="password">
|
||||
<a-input-password
|
||||
v-model:value="formState.password"
|
||||
placeholder="请输入密码"
|
||||
autocomplete="current-password"
|
||||
size="large"
|
||||
@pressEnter="handleSubmit"
|
||||
>
|
||||
<template #prefix><LockOutlined /></template>
|
||||
</a-input-password>
|
||||
</a-form-item>
|
||||
<a-form-item>
|
||||
<a-button
|
||||
type="primary"
|
||||
html-type="submit"
|
||||
block
|
||||
size="large"
|
||||
:loading="isLoading"
|
||||
>
|
||||
登录
|
||||
</a-button>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
<div class="login-page__footer">
|
||||
QMDSearch Admin · 分层信息检索服务
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.login-page {
|
||||
position: relative;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
overflow: hidden;
|
||||
background: linear-gradient(135deg, #1e3a8a 0%, #1e40af 35%, #0c4a6e 100%);
|
||||
}
|
||||
|
||||
.login-page__bg-deco {
|
||||
position: absolute;
|
||||
border-radius: 50%;
|
||||
filter: blur(80px);
|
||||
opacity: 0.4;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.login-page__bg-deco--1 {
|
||||
width: 360px;
|
||||
height: 360px;
|
||||
background: #4096ff;
|
||||
top: -120px;
|
||||
right: -80px;
|
||||
}
|
||||
|
||||
.login-page__bg-deco--2 {
|
||||
width: 320px;
|
||||
height: 320px;
|
||||
background: #13c2c2;
|
||||
bottom: -100px;
|
||||
left: -60px;
|
||||
}
|
||||
|
||||
.login-page__box {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
width: 380px;
|
||||
background: #fff;
|
||||
border-radius: 12px;
|
||||
padding: 32px 36px 24px;
|
||||
box-shadow: 0 16px 48px rgba(0, 0, 0, 0.25);
|
||||
}
|
||||
|
||||
.login-page__brand {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
margin-bottom: 28px;
|
||||
}
|
||||
|
||||
.login-page__logo {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
border-radius: 10px;
|
||||
background: linear-gradient(135deg, #1677ff 0%, #4096ff 100%);
|
||||
color: #fff;
|
||||
font-size: 26px;
|
||||
box-shadow: 0 6px 16px rgba(22, 119, 255, 0.4);
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.login-page__brand-text {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.login-page__brand-name {
|
||||
font-size: 22px;
|
||||
font-weight: 700;
|
||||
color: #1f2937;
|
||||
line-height: 1.1;
|
||||
}
|
||||
|
||||
.login-page__brand-sub {
|
||||
font-size: 13px;
|
||||
color: #6b7280;
|
||||
}
|
||||
|
||||
.login-page__footer {
|
||||
text-align: center;
|
||||
font-size: 12px;
|
||||
color: #9ca3af;
|
||||
margin-top: 8px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,349 @@
|
||||
<script setup>
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
import { message } from 'ant-design-vue'
|
||||
import {
|
||||
DatabaseOutlined,
|
||||
ApartmentOutlined,
|
||||
FileSearchOutlined,
|
||||
BlockOutlined,
|
||||
FileTextOutlined,
|
||||
QuestionCircleOutlined,
|
||||
ReloadOutlined,
|
||||
AppstoreOutlined
|
||||
} from '@ant-design/icons-vue'
|
||||
import { stats as fetchStats } from '@/api/knowledge'
|
||||
|
||||
const isLoading = ref(false)
|
||||
const statsData = ref(null)
|
||||
|
||||
const cardMeta = [
|
||||
{
|
||||
key: 'doc_l1',
|
||||
label: 'L1 文档总结',
|
||||
icon: DatabaseOutlined,
|
||||
color: '#1677ff',
|
||||
bg: 'rgba(22, 119, 255, 0.12)'
|
||||
},
|
||||
{
|
||||
key: 'doc_l2',
|
||||
label: 'L2 大纲节点',
|
||||
icon: ApartmentOutlined,
|
||||
color: '#722ed1',
|
||||
bg: 'rgba(114, 46, 209, 0.12)'
|
||||
},
|
||||
{
|
||||
key: 'doc_l3',
|
||||
label: 'L3 内容大纲',
|
||||
icon: FileSearchOutlined,
|
||||
color: '#13c2c2',
|
||||
bg: 'rgba(19, 194, 194, 0.12)'
|
||||
},
|
||||
{
|
||||
key: 'chunks',
|
||||
label: 'Chunks',
|
||||
icon: BlockOutlined,
|
||||
color: '#fa8c16',
|
||||
bg: 'rgba(250, 140, 22, 0.12)'
|
||||
},
|
||||
{
|
||||
key: '__documents_total',
|
||||
label: '文档总数',
|
||||
icon: FileTextOutlined,
|
||||
color: '#52c41a',
|
||||
bg: 'rgba(82, 196, 26, 0.12)'
|
||||
},
|
||||
{
|
||||
key: '__uncategorized',
|
||||
label: '未分类文档',
|
||||
icon: QuestionCircleOutlined,
|
||||
color: '#ff4d4f',
|
||||
bg: 'rgba(255, 77, 79, 0.12)'
|
||||
}
|
||||
]
|
||||
|
||||
const cards = computed(() => {
|
||||
const collections = statsData.value?.collections || {}
|
||||
return cardMeta.map((m) => {
|
||||
let value = 0
|
||||
if (m.key === '__documents_total') {
|
||||
value = statsData.value?.documents_total ?? 0
|
||||
} else if (m.key === '__uncategorized') {
|
||||
value = statsData.value?.uncategorized_count ?? 0
|
||||
} else {
|
||||
value = collections[m.key] ?? 0
|
||||
}
|
||||
return { ...m, value }
|
||||
})
|
||||
})
|
||||
|
||||
const categoryBars = computed(() => {
|
||||
const categories = statsData.value?.categories || {}
|
||||
const entries = Object.entries(categories).map(([name, count]) => ({ name, count }))
|
||||
entries.sort((a, b) => b.count - a.count)
|
||||
const max = entries.reduce((acc, item) => Math.max(acc, item.count), 1)
|
||||
const total = entries.reduce((acc, item) => acc + item.count, 0)
|
||||
return entries.map((item) => ({
|
||||
...item,
|
||||
percent: Math.round((item.count / max) * 100),
|
||||
ratio: total ? Math.round((item.count / total) * 100) : 0
|
||||
}))
|
||||
})
|
||||
|
||||
const gradientColors = [
|
||||
'linear-gradient(90deg, #1677ff 0%, #4096ff 100%)',
|
||||
'linear-gradient(90deg, #722ed1 0%, #9254de 100%)',
|
||||
'linear-gradient(90deg, #13c2c2 0%, #36cfc9 100%)',
|
||||
'linear-gradient(90deg, #fa8c16 0%, #ffa940 100%)',
|
||||
'linear-gradient(90deg, #52c41a 0%, #73d13d 100%)',
|
||||
'linear-gradient(90deg, #eb2f96 0%, #f759ab 100%)',
|
||||
'linear-gradient(90deg, #fa541c 0%, #ff7a45 100%)',
|
||||
'linear-gradient(90deg, #2f54eb 0%, #597ef7 100%)',
|
||||
'linear-gradient(90deg, #08979c 0%, #13c2c2 100%)',
|
||||
'linear-gradient(90deg, #c41d7f 0%, #eb2f96 100%)'
|
||||
]
|
||||
|
||||
function barGradient(index) {
|
||||
return gradientColors[index % gradientColors.length]
|
||||
}
|
||||
|
||||
async function loadStats() {
|
||||
isLoading.value = true
|
||||
try {
|
||||
statsData.value = await fetchStats()
|
||||
} catch (err) {
|
||||
message.error(err?.message || '获取统计失败')
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
loadStats()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="overview page-section">
|
||||
<div class="overview__header">
|
||||
<h2 class="page-title">概览</h2>
|
||||
<a-space>
|
||||
<a-button :loading="isLoading" @click="loadStats">
|
||||
<template #icon><ReloadOutlined /></template>
|
||||
刷新
|
||||
</a-button>
|
||||
</a-space>
|
||||
</div>
|
||||
|
||||
<a-spin :spinning="isLoading">
|
||||
<div class="overview__cards">
|
||||
<div
|
||||
v-for="card in cards"
|
||||
:key="card.key"
|
||||
class="overview__card"
|
||||
>
|
||||
<div class="overview__card-body">
|
||||
<div
|
||||
class="overview__card-icon"
|
||||
:style="{ background: card.bg, color: card.color }"
|
||||
>
|
||||
<component :is="card.icon" />
|
||||
</div>
|
||||
<div class="overview__card-info">
|
||||
<div class="overview__card-num">{{ card.value }}</div>
|
||||
<div class="overview__card-label">{{ card.label }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="overview__section">
|
||||
<div class="overview__section-head">
|
||||
<h3 class="overview__subtitle">
|
||||
<AppstoreOutlined />
|
||||
<span>类目分布</span>
|
||||
</h3>
|
||||
<span v-if="categoryBars.length" class="text-muted overview__section-meta">
|
||||
共 {{ categoryBars.length }} 个类目
|
||||
</span>
|
||||
</div>
|
||||
<div v-if="categoryBars.length === 0" class="overview__empty text-muted">
|
||||
暂无数据
|
||||
</div>
|
||||
<div v-else class="overview__bars">
|
||||
<div
|
||||
v-for="(bar, idx) in categoryBars"
|
||||
:key="bar.name"
|
||||
class="overview__bar-row"
|
||||
:title="`${bar.name}: ${bar.count} (${bar.ratio}%)`"
|
||||
>
|
||||
<span class="overview__bar-name" :title="bar.name">{{ bar.name }}</span>
|
||||
<div class="overview__bar-track">
|
||||
<div
|
||||
class="overview__bar-fill"
|
||||
:style="{ width: bar.percent + '%', background: barGradient(idx) }"
|
||||
/>
|
||||
</div>
|
||||
<span class="overview__bar-count">{{ bar.count }}</span>
|
||||
<span class="overview__bar-ratio">{{ bar.ratio }}%</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</a-spin>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.overview__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.overview__cards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
|
||||
gap: 16px;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
.overview__card {
|
||||
background: #fff;
|
||||
border: 1px solid #f0f0f0;
|
||||
border-radius: 8px;
|
||||
padding: 16px 18px;
|
||||
transition: all 0.2s;
|
||||
box-shadow: 0 1px 2px rgba(0, 21, 41, 0.04);
|
||||
}
|
||||
|
||||
.overview__card:hover {
|
||||
box-shadow: 0 4px 12px rgba(0, 21, 41, 0.08);
|
||||
transform: translateY(-2px);
|
||||
}
|
||||
|
||||
.overview__card-body {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.overview__card-icon {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
border-radius: 8px;
|
||||
font-size: 22px;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.overview__card-info {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.overview__card-num {
|
||||
font-size: 26px;
|
||||
font-weight: 700;
|
||||
color: #1f2937;
|
||||
line-height: 1.1;
|
||||
}
|
||||
|
||||
.overview__card-label {
|
||||
font-size: 12px;
|
||||
color: #6b7280;
|
||||
}
|
||||
|
||||
.overview__section {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.overview__section-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.overview__subtitle {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #374151;
|
||||
margin: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.overview__section-meta {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.overview__empty {
|
||||
text-align: center;
|
||||
padding: 32px 0;
|
||||
}
|
||||
|
||||
.overview__bars {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.overview__bar-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
font-size: 13px;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.overview__bar-name {
|
||||
width: 180px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
color: #374151;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.overview__bar-track {
|
||||
flex: 1;
|
||||
background: #f3f4f6;
|
||||
border-radius: 4px;
|
||||
height: 18px;
|
||||
margin: 0 10px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.overview__bar-fill {
|
||||
height: 100%;
|
||||
border-radius: 4px;
|
||||
min-width: 2px;
|
||||
transition: width 0.4s ease;
|
||||
}
|
||||
|
||||
.overview__bar-count {
|
||||
width: 48px;
|
||||
text-align: right;
|
||||
color: #1f2937;
|
||||
font-weight: 600;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.overview__bar-ratio {
|
||||
width: 48px;
|
||||
text-align: right;
|
||||
color: #6b7280;
|
||||
font-size: 12px;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.overview__bar-name {
|
||||
width: 120px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,234 @@
|
||||
<script setup>
|
||||
import { reactive, ref } from 'vue'
|
||||
import { message } from 'ant-design-vue'
|
||||
import { search as searchApi } from '@/api/search'
|
||||
|
||||
const isLoading = ref(false)
|
||||
const resultData = ref(null)
|
||||
|
||||
const formRef = ref(null)
|
||||
const formState = reactive({
|
||||
query: '',
|
||||
top_k: 5,
|
||||
summarize: false
|
||||
})
|
||||
|
||||
const rules = {
|
||||
query: [{ required: true, message: '请输入查询语句', trigger: 'blur' }],
|
||||
top_k: [{ type: 'number', min: 1, max: 50, message: 'top_k 范围 1~50', trigger: 'change' }]
|
||||
}
|
||||
|
||||
const routedCategoriesText = (cats) => {
|
||||
if (!cats || !cats.length) return '(无)'
|
||||
return cats.join(', ')
|
||||
}
|
||||
|
||||
async function handleSearch() {
|
||||
try {
|
||||
await formRef.value.validate()
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
isLoading.value = true
|
||||
resultData.value = null
|
||||
try {
|
||||
const payload = {
|
||||
query: formState.query,
|
||||
top_k: formState.top_k,
|
||||
summarize: formState.summarize
|
||||
}
|
||||
resultData.value = await searchApi(payload)
|
||||
} catch (err) {
|
||||
message.error(err?.message || '检索失败')
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function eiValue(value) {
|
||||
if (value === undefined || value === null || value === '') return '(无)'
|
||||
if (Array.isArray(value)) {
|
||||
return value.length ? value.join(', ') : '(无)'
|
||||
}
|
||||
return String(value)
|
||||
}
|
||||
|
||||
const hits = (data) => data?.hits || []
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="search page-section">
|
||||
<div class="search__header">
|
||||
<h2 class="search__title">检索测试台</h2>
|
||||
</div>
|
||||
|
||||
<a-form
|
||||
ref="formRef"
|
||||
:model="formState"
|
||||
:rules="rules"
|
||||
layout="vertical"
|
||||
>
|
||||
<a-form-item label="查询语句" name="query">
|
||||
<a-input
|
||||
v-model:value="formState.query"
|
||||
placeholder="请输入查询语句"
|
||||
allow-clear
|
||||
@pressEnter="handleSearch"
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item label="top_k" name="top_k">
|
||||
<a-input-number
|
||||
v-model:value="formState.top_k"
|
||||
:min="1"
|
||||
:max="50"
|
||||
style="width: 160px"
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item name="summarize">
|
||||
<a-checkbox v-model:checked="formState.summarize">
|
||||
对结果生成 AI 总结
|
||||
</a-checkbox>
|
||||
</a-form-item>
|
||||
<a-form-item>
|
||||
<a-button type="primary" :loading="isLoading" @click="handleSearch">
|
||||
检索
|
||||
</a-button>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
|
||||
<div v-if="resultData" class="search__result">
|
||||
<a-descriptions :column="1" size="small" bordered>
|
||||
<a-descriptions-item label="routed_categories">
|
||||
{{ routedCategoriesText(resultData.routed_categories) }}
|
||||
</a-descriptions-item>
|
||||
</a-descriptions>
|
||||
|
||||
<a-alert
|
||||
v-if="resultData.fallback"
|
||||
class="search__alert"
|
||||
type="warning"
|
||||
show-icon
|
||||
message="fallback:本次检索触发了回退策略(路由类目无命中,已降级全局检索)"
|
||||
/>
|
||||
|
||||
<div v-if="resultData.summary" class="search__summary">
|
||||
<div class="search__summary-title">AI 总结</div>
|
||||
<div class="text-break">{{ resultData.summary }}</div>
|
||||
</div>
|
||||
|
||||
<div v-if="resultData.extracted_info" class="search__extracted">
|
||||
<div class="search__extracted-title">AI 提取的关键信息</div>
|
||||
<a-descriptions :column="1" size="small" bordered>
|
||||
<a-descriptions-item label="改写">
|
||||
{{ eiValue(resultData.extracted_info.rewrite) }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="关键词">
|
||||
{{ eiValue(resultData.extracted_info.keywords) }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="实体">
|
||||
{{ eiValue(resultData.extracted_info.entities) }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="意图">
|
||||
{{ eiValue(resultData.extracted_info.intent) }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="时间范围">
|
||||
{{ eiValue(resultData.extracted_info.time_range) }}
|
||||
</a-descriptions-item>
|
||||
<a-descriptions-item label="命中类目">
|
||||
{{ eiValue(resultData.extracted_info.categories) }}
|
||||
</a-descriptions-item>
|
||||
</a-descriptions>
|
||||
</div>
|
||||
|
||||
<div class="text-muted" style="margin: 12px 0 8px">
|
||||
命中 {{ hits(resultData).length }} 条
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-for="(hit, idx) in hits(resultData)"
|
||||
:key="`hit-${idx}`"
|
||||
class="search__hit"
|
||||
>
|
||||
<div class="search__hit-meta">
|
||||
score={{ hit.score }} | doc_id={{ hit.doc_id }} | 标题={{ hit.title || '' }} | section={{ hit.section_path || '' }}
|
||||
</div>
|
||||
<div class="search__hit-snippet text-break">{{ hit.text || '' }}</div>
|
||||
<div v-if="hit.doc_summary" class="search__hit-meta">
|
||||
文档摘要:{{ hit.doc_summary }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.search__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.search__title {
|
||||
font-size: 16px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.search__result {
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.search__alert {
|
||||
margin: 12px 0;
|
||||
}
|
||||
|
||||
.search__summary {
|
||||
background: #f0fdf4;
|
||||
border: 1px solid #bbf7d0;
|
||||
border-radius: 6px;
|
||||
padding: 12px 14px;
|
||||
margin: 12px 0;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.search__summary-title {
|
||||
font-weight: 600;
|
||||
margin-bottom: 6px;
|
||||
color: #15803d;
|
||||
}
|
||||
|
||||
.search__extracted {
|
||||
background: #f0f9ff;
|
||||
border: 1px solid #bae6fd;
|
||||
border-radius: 6px;
|
||||
padding: 12px 14px;
|
||||
margin: 12px 0;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.search__extracted-title {
|
||||
font-weight: 600;
|
||||
margin-bottom: 8px;
|
||||
color: #0369a1;
|
||||
}
|
||||
|
||||
.search__hit {
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 6px;
|
||||
padding: 10px 12px;
|
||||
margin-bottom: 10px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.search__hit-meta {
|
||||
font-size: 12px;
|
||||
color: #6b7280;
|
||||
margin-bottom: 6px;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.search__hit-snippet {
|
||||
font-size: 13px;
|
||||
color: #1f2937;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,489 @@
|
||||
<script setup>
|
||||
import { computed, onMounted, reactive, ref } from 'vue'
|
||||
import { message, Modal } from 'ant-design-vue'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import {
|
||||
RobotOutlined,
|
||||
FileTextOutlined,
|
||||
FilterOutlined,
|
||||
ReloadOutlined,
|
||||
SaveOutlined,
|
||||
UndoOutlined,
|
||||
LockOutlined
|
||||
} from '@ant-design/icons-vue'
|
||||
import {
|
||||
get as getSettings,
|
||||
update as updateSettings,
|
||||
schema as getSettingsSchema,
|
||||
reset as resetSettings
|
||||
} from '@/api/settings'
|
||||
import { useAuthStore } from '@/stores/useAuthStore'
|
||||
|
||||
const authStore = useAuthStore()
|
||||
const { isAdmin } = storeToRefs(authStore)
|
||||
|
||||
const activeTab = ref('models')
|
||||
const isLoading = ref(false)
|
||||
const isSaving = ref(false)
|
||||
const isResetting = ref(false)
|
||||
const schemaData = ref(null)
|
||||
|
||||
const DEFAULT_MODEL_CONFIG = {
|
||||
provider: 'ollama',
|
||||
base_url: '',
|
||||
api_key: '',
|
||||
model: '',
|
||||
timeout: 120,
|
||||
temperature: 0.3
|
||||
}
|
||||
|
||||
const form = reactive({
|
||||
models: {
|
||||
summarize: { ...DEFAULT_MODEL_CONFIG },
|
||||
query: { ...DEFAULT_MODEL_CONFIG },
|
||||
classify: { ...DEFAULT_MODEL_CONFIG }
|
||||
},
|
||||
parsers: {
|
||||
ocr: { plugin: '', params: {} },
|
||||
pdf: { plugin: '', params: {} },
|
||||
docx: { plugin: '', params: {} }
|
||||
},
|
||||
dedup: {
|
||||
strategy: 'sha256',
|
||||
simhash_threshold: 3,
|
||||
ttl_seconds: 86400
|
||||
}
|
||||
})
|
||||
|
||||
const modelKeys = ['summarize', 'query', 'classify']
|
||||
const modelMeta = {
|
||||
summarize: {
|
||||
label: '总结模型',
|
||||
key: 'summarize',
|
||||
desc: '用于文档三级总结',
|
||||
icon: RobotOutlined,
|
||||
color: '#1677ff'
|
||||
},
|
||||
query: {
|
||||
label: '查询模型',
|
||||
key: 'query',
|
||||
desc: '用于 query 解析与重写',
|
||||
icon: FilterOutlined,
|
||||
color: '#722ed1'
|
||||
},
|
||||
classify: {
|
||||
label: '分类模型',
|
||||
key: 'classify',
|
||||
desc: '用于文档分类判定',
|
||||
icon: FileTextOutlined,
|
||||
color: '#13c2c2'
|
||||
}
|
||||
}
|
||||
|
||||
const llmProviderOptions = computed(() => {
|
||||
return (schemaData.value?.llm_providers || ['ollama', 'openai_compatible']).map(
|
||||
(p) => ({ label: p, value: p })
|
||||
)
|
||||
})
|
||||
|
||||
const ocrPluginOptions = computed(() =>
|
||||
(schemaData.value?.ocr_plugins || []).map((p) => ({ label: p, value: p }))
|
||||
)
|
||||
const pdfPluginOptions = computed(() =>
|
||||
(schemaData.value?.pdf_plugins || []).map((p) => ({ label: p, value: p }))
|
||||
)
|
||||
const docxPluginOptions = computed(() =>
|
||||
(schemaData.value?.docx_plugins || []).map((p) => ({ label: p, value: p }))
|
||||
)
|
||||
const dedupStrategyOptions = computed(() =>
|
||||
(schemaData.value?.dedup_strategies || ['none', 'sha256', 'simhash']).map((s) => ({
|
||||
label: s,
|
||||
value: s
|
||||
}))
|
||||
)
|
||||
|
||||
const isSimhash = computed(() => form.dedup.strategy === 'simhash')
|
||||
|
||||
function applySettingsToForm(cfg) {
|
||||
if (!cfg || typeof cfg !== 'object') return
|
||||
if (cfg.models && typeof cfg.models === 'object') {
|
||||
for (const key of modelKeys) {
|
||||
const src = cfg.models[key] || {}
|
||||
form.models[key] = {
|
||||
provider: src.provider ?? DEFAULT_MODEL_CONFIG.provider,
|
||||
base_url: src.base_url ?? '',
|
||||
api_key: src.api_key ?? '',
|
||||
model: src.model ?? '',
|
||||
timeout: src.timeout ?? 120,
|
||||
temperature: src.temperature ?? 0.3
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cfg.parsers && typeof cfg.parsers === 'object') {
|
||||
const ocr = cfg.parsers.ocr || {}
|
||||
const pdf = cfg.parsers.pdf || {}
|
||||
const docx = cfg.parsers.docx || {}
|
||||
form.parsers.ocr = { plugin: ocr.plugin ?? '', params: ocr.params || {} }
|
||||
form.parsers.pdf = { plugin: pdf.plugin ?? '', params: pdf.params || {} }
|
||||
form.parsers.docx = { plugin: docx.plugin ?? '', params: docx.params || {} }
|
||||
}
|
||||
if (cfg.dedup && typeof cfg.dedup === 'object') {
|
||||
form.dedup.strategy = cfg.dedup.strategy ?? 'sha256'
|
||||
form.dedup.simhash_threshold = cfg.dedup.simhash_threshold ?? 3
|
||||
form.dedup.ttl_seconds = cfg.dedup.ttl_seconds ?? 86400
|
||||
}
|
||||
}
|
||||
|
||||
function buildPayload() {
|
||||
return {
|
||||
models: {
|
||||
summarize: { ...form.models.summarize },
|
||||
query: { ...form.models.query },
|
||||
classify: { ...form.models.classify }
|
||||
},
|
||||
parsers: {
|
||||
ocr: { plugin: form.parsers.ocr.plugin, params: { ...form.parsers.ocr.params } },
|
||||
pdf: { plugin: form.parsers.pdf.plugin, params: { ...form.parsers.pdf.params } },
|
||||
docx: { plugin: form.parsers.docx.plugin, params: { ...form.parsers.docx.params } }
|
||||
},
|
||||
dedup: {
|
||||
strategy: form.dedup.strategy,
|
||||
simhash_threshold: form.dedup.simhash_threshold,
|
||||
ttl_seconds: form.dedup.ttl_seconds
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function loadAll() {
|
||||
isLoading.value = true
|
||||
try {
|
||||
const [cfg, schema] = await Promise.all([getSettings(), getSettingsSchema()])
|
||||
schemaData.value = schema
|
||||
applySettingsToForm(cfg)
|
||||
} catch (err) {
|
||||
message.error(err?.message || '加载设置失败')
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSave() {
|
||||
if (!isAdmin.value) {
|
||||
message.error('仅管理员可修改设置')
|
||||
return
|
||||
}
|
||||
isSaving.value = true
|
||||
try {
|
||||
const payload = buildPayload()
|
||||
const cfg = await updateSettings(payload)
|
||||
applySettingsToForm(cfg)
|
||||
message.success('设置已保存')
|
||||
} catch (err) {
|
||||
message.error(err?.message || '保存设置失败')
|
||||
} finally {
|
||||
isSaving.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function handleReset() {
|
||||
if (!isAdmin.value) {
|
||||
message.error('仅管理员可重置设置')
|
||||
return
|
||||
}
|
||||
Modal.confirm({
|
||||
title: '确认重置',
|
||||
content: '确定要将所有运行时设置重置为默认值吗?此操作不可撤销。',
|
||||
okText: '重置',
|
||||
okType: 'danger',
|
||||
cancelText: '取消',
|
||||
async onOk() {
|
||||
isResetting.value = true
|
||||
try {
|
||||
const cfg = await resetSettings()
|
||||
applySettingsToForm(cfg)
|
||||
message.success('已重置为默认值')
|
||||
} catch (err) {
|
||||
message.error(err?.message || '重置失败')
|
||||
} finally {
|
||||
isResetting.value = false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
loadAll()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="settings page-section">
|
||||
<div class="settings__header">
|
||||
<h2 class="page-title">设置</h2>
|
||||
<a-space>
|
||||
<a-button :loading="isLoading" @click="loadAll">
|
||||
<template #icon><ReloadOutlined /></template>
|
||||
刷新
|
||||
</a-button>
|
||||
<a-button
|
||||
type="primary"
|
||||
:loading="isSaving"
|
||||
:disabled="!isAdmin"
|
||||
@click="handleSave"
|
||||
>
|
||||
<template #icon><SaveOutlined /></template>
|
||||
保存
|
||||
</a-button>
|
||||
<a-button
|
||||
danger
|
||||
:loading="isResetting"
|
||||
:disabled="!isAdmin"
|
||||
@click="handleReset"
|
||||
>
|
||||
<template #icon><UndoOutlined /></template>
|
||||
重置默认
|
||||
</a-button>
|
||||
</a-space>
|
||||
</div>
|
||||
|
||||
<a-alert
|
||||
v-if="!isAdmin"
|
||||
class="settings__alert"
|
||||
type="info"
|
||||
show-icon
|
||||
message="当前用户非管理员,仅可查看设置;保存与重置需 admin 权限。"
|
||||
/>
|
||||
|
||||
<a-spin :spinning="isLoading">
|
||||
<a-tabs v-model:activeKey="activeTab">
|
||||
<a-tab-pane key="models" tab="模型配置">
|
||||
<div class="settings__cards">
|
||||
<a-card
|
||||
v-for="key in modelKeys"
|
||||
:key="key"
|
||||
class="settings__card"
|
||||
:body-style="{ padding: '16px 18px' }"
|
||||
>
|
||||
<template #title>
|
||||
<div class="settings__card-title">
|
||||
<span
|
||||
class="settings__card-icon"
|
||||
:style="{ background: modelMeta[key].color }"
|
||||
>
|
||||
<component :is="modelMeta[key].icon" />
|
||||
</span>
|
||||
<div class="settings__card-head">
|
||||
<div class="settings__card-name">{{ modelMeta[key].label }}</div>
|
||||
<div class="settings__card-desc">{{ modelMeta[key].desc }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<a-form layout="vertical" :colon="false">
|
||||
<a-form-item label="provider">
|
||||
<a-radio-group v-model:value="form.models[key].provider" button-style="solid">
|
||||
<a-radio
|
||||
v-for="opt in llmProviderOptions"
|
||||
:key="opt.value"
|
||||
:value="opt.value"
|
||||
>
|
||||
{{ opt.label }}
|
||||
</a-radio>
|
||||
</a-radio-group>
|
||||
</a-form-item>
|
||||
<a-row :gutter="12">
|
||||
<a-col :span="24">
|
||||
<a-form-item label="base_url">
|
||||
<a-input
|
||||
v-model:value="form.models[key].base_url"
|
||||
placeholder="例如 http://localhost:11434"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-col>
|
||||
</a-row>
|
||||
<a-form-item
|
||||
v-if="form.models[key].provider !== 'ollama'"
|
||||
label="api_key"
|
||||
>
|
||||
<a-input-password
|
||||
v-model:value="form.models[key].api_key"
|
||||
placeholder="无则留空"
|
||||
allow-clear
|
||||
>
|
||||
<template #prefix><LockOutlined /></template>
|
||||
</a-input-password>
|
||||
</a-form-item>
|
||||
<a-row :gutter="12">
|
||||
<a-col :span="14">
|
||||
<a-form-item label="model">
|
||||
<a-input
|
||||
v-model:value="form.models[key].model"
|
||||
placeholder="例如 qwen2.5:1.5b"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-col>
|
||||
<a-col :span="10">
|
||||
<a-form-item label="timeout (秒)">
|
||||
<a-input-number
|
||||
v-model:value="form.models[key].timeout"
|
||||
:min="1"
|
||||
style="width: 100%"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-col>
|
||||
</a-row>
|
||||
<a-form-item label="temperature">
|
||||
<a-input-number
|
||||
v-model:value="form.models[key].temperature"
|
||||
:min="0"
|
||||
:max="2"
|
||||
:step="0.1"
|
||||
style="width: 100%"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</a-card>
|
||||
</div>
|
||||
</a-tab-pane>
|
||||
|
||||
<a-tab-pane key="parsers" tab="解析插件">
|
||||
<a-form layout="vertical" class="settings__parsers">
|
||||
<a-form-item label="OCR 插件">
|
||||
<a-select
|
||||
v-model:value="form.parsers.ocr.plugin"
|
||||
:options="ocrPluginOptions"
|
||||
placeholder="选择 OCR 插件"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item label="PDF 插件">
|
||||
<a-select
|
||||
v-model:value="form.parsers.pdf.plugin"
|
||||
:options="pdfPluginOptions"
|
||||
placeholder="选择 PDF 插件"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item label="DOCX 插件">
|
||||
<a-select
|
||||
v-model:value="form.parsers.docx.plugin"
|
||||
:options="docxPluginOptions"
|
||||
placeholder="选择 DOCX 插件"
|
||||
allow-clear
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</a-tab-pane>
|
||||
|
||||
<a-tab-pane key="dedup" tab="去重策略">
|
||||
<a-form layout="vertical" class="settings__dedup">
|
||||
<a-form-item label="strategy">
|
||||
<a-select
|
||||
v-model:value="form.dedup.strategy"
|
||||
:options="dedupStrategyOptions"
|
||||
placeholder="选择去重策略"
|
||||
/>
|
||||
</a-form-item>
|
||||
<a-form-item label="simhash_threshold (0~64)">
|
||||
<a-input-number
|
||||
v-model:value="form.dedup.simhash_threshold"
|
||||
:min="0"
|
||||
:max="64"
|
||||
:disabled="!isSimhash"
|
||||
style="width: 100%"
|
||||
/>
|
||||
<span v-if="!isSimhash" class="text-muted" style="margin-top: 4px; display: inline-block">
|
||||
仅 strategy=simhash 时启用
|
||||
</span>
|
||||
</a-form-item>
|
||||
<a-form-item label="ttl_seconds">
|
||||
<a-input-number
|
||||
v-model:value="form.dedup.ttl_seconds"
|
||||
:min="0"
|
||||
style="width: 100%"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</a-tab-pane>
|
||||
</a-tabs>
|
||||
</a-spin>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.settings__header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.settings__alert {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.settings__cards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(320px, 1fr));
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.settings__card {
|
||||
border-radius: 8px;
|
||||
border: 1px solid #e5e7eb;
|
||||
transition: box-shadow 0.2s;
|
||||
}
|
||||
|
||||
.settings__card:hover {
|
||||
box-shadow: 0 4px 12px rgba(0, 21, 41, 0.06);
|
||||
}
|
||||
|
||||
.settings__card :deep(.ant-card-head) {
|
||||
padding: 12px 16px;
|
||||
border-bottom: 1px solid #f0f0f0;
|
||||
min-height: auto;
|
||||
}
|
||||
|
||||
.settings__card-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.settings__card-icon {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
border-radius: 6px;
|
||||
color: #fff;
|
||||
font-size: 16px;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
.settings__card-head {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.settings__card-name {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #1f2937;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
.settings__card-desc {
|
||||
font-size: 12px;
|
||||
color: #6b7280;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
.settings__parsers,
|
||||
.settings__dedup {
|
||||
max-width: 480px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,32 @@
|
||||
import { fileURLToPath, URL } from 'node:url'
|
||||
import { defineConfig } from 'vite'
|
||||
import vue from '@vitejs/plugin-vue'
|
||||
|
||||
// Vite 配置:base 部署到 /admin/,dev 下 /api 与 /admin 代理到后端 8000
|
||||
export default defineConfig({
|
||||
base: '/admin/',
|
||||
plugins: [vue()],
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': fileURLToPath(new URL('./src', import.meta.url))
|
||||
}
|
||||
},
|
||||
server: {
|
||||
port: 5173,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8000',
|
||||
changeOrigin: true
|
||||
},
|
||||
'/admin': {
|
||||
target: 'http://localhost:8000',
|
||||
changeOrigin: true
|
||||
}
|
||||
}
|
||||
},
|
||||
build: {
|
||||
outDir: 'dist',
|
||||
sourcemap: false,
|
||||
chunkSizeWarningLimit: 1500
|
||||
}
|
||||
})
|
||||
@@ -5,6 +5,9 @@ DELETE /documents 加了 Depends(require_admin)。这里通过 autouse 夹具把
|
||||
统一替换为返回固定 admin AuthUser 的 lambda,使现有 API 测试无需改动即可通过认证。
|
||||
单个测试需要走真实认证逻辑时(如 tests/test_auth.py),可在测试函数内
|
||||
pop 掉对应 override,autouse fixture yield 后会统一 clear。
|
||||
|
||||
另外清理 LLM 客户端 / 解析插件 / 去重策略三处进程级缓存,避免跨测试串扰
|
||||
(不同测试可能注入不同的 redis/ollama 实例,缓存按配置签名而非实例区分)。
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
@@ -25,3 +28,24 @@ def override_auth():
|
||||
app.dependency_overrides[require_admin] = lambda: TEST_USER
|
||||
yield
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _invalidate_runtime_caches():
|
||||
"""每个测试前后清理 LLM/解析插件/去重策略进程级缓存
|
||||
|
||||
这三处缓存按 runtime_settings 配置签名而非实例区分,跨测试若配置相同
|
||||
会复用旧实例(绑定到上个测试的 redis/ollama 替身),导致串扰。
|
||||
"""
|
||||
# 延迟导入避免循环依赖
|
||||
from app.core.dedup import invalidate_dedup_strategy_cache
|
||||
from app.core.file_parser import invalidate_parser_plugin_cache
|
||||
from app.services.llm import invalidate_llm_client_cache
|
||||
|
||||
invalidate_llm_client_cache()
|
||||
invalidate_parser_plugin_cache()
|
||||
invalidate_dedup_strategy_cache()
|
||||
yield
|
||||
invalidate_llm_client_cache()
|
||||
invalidate_parser_plugin_cache()
|
||||
invalidate_dedup_strategy_cache()
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
"""文本去重策略单元测试:none / sha256 / simhash 三种策略 lookup+record + 工厂
|
||||
|
||||
使用内存版 FakeRedis,不连真实 Redis。
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core import runtime_settings as rs
|
||||
from app.core.dedup import (
|
||||
DEDUP_KEY_PREFIX,
|
||||
NoopDedupStrategy,
|
||||
Sha256DedupStrategy,
|
||||
SimhashDedupStrategy,
|
||||
_hamming_distance,
|
||||
_simhash,
|
||||
compute_text_hash,
|
||||
get_dedup_strategy,
|
||||
invalidate_dedup_strategy_cache,
|
||||
)
|
||||
|
||||
|
||||
class FakeRedis:
|
||||
"""内存版 Redis:实现 get_json/set_json,可记录所有写入"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.store: dict[str, dict[str, Any]] = {}
|
||||
self.writes: list[tuple[str, dict[str, Any], int | None]] = []
|
||||
|
||||
async def get_json(self, key: str) -> dict[str, Any] | None:
|
||||
return self.store.get(key)
|
||||
|
||||
async def set_json(self, key: str, value: dict[str, Any], ttl: int | None = None) -> bool:
|
||||
self.writes.append((key, value, ttl))
|
||||
self.store[key] = value
|
||||
return True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_settings_path(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
path = tmp_path / "runtime_settings.json"
|
||||
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
|
||||
rs._runtime_settings = None
|
||||
invalidate_dedup_strategy_cache()
|
||||
yield path
|
||||
rs._runtime_settings = None
|
||||
invalidate_dedup_strategy_cache()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工具函数
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestSimhashUtil:
|
||||
def test_empty_text_returns_zero(self):
|
||||
assert _simhash("") == 0
|
||||
|
||||
def test_same_text_same_fingerprint(self):
|
||||
assert _simhash("同一段文本") == _simhash("同一段文本")
|
||||
|
||||
def test_different_text_different_fingerprint(self):
|
||||
assert _simhash("文本A") != _simhash("文本B是完全不同的内容")
|
||||
|
||||
def test_hamming_distance_zero_for_same(self):
|
||||
fp = _simhash("hello")
|
||||
assert _hamming_distance(fp, fp) == 0
|
||||
|
||||
def test_hamming_distance_count(self):
|
||||
# 0b001 vs 0b100 → 两个位不同
|
||||
assert _hamming_distance(0b001, 0b100) == 2
|
||||
|
||||
def test_compute_text_hash_is_sha256_hex(self):
|
||||
import hashlib
|
||||
|
||||
text = "abc"
|
||||
assert compute_text_hash(text) == hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# NoopDedupStrategy
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestNoopStrategy:
|
||||
async def test_lookup_always_none(self):
|
||||
s = NoopDedupStrategy()
|
||||
assert await s.lookup("any") is None
|
||||
|
||||
async def test_record_does_nothing(self):
|
||||
s = NoopDedupStrategy()
|
||||
await s.record("any", {"x": 1}) # 不抛异常即可
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Sha256DedupStrategy
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestSha256Strategy:
|
||||
def test_key_format(self):
|
||||
s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100)
|
||||
key = s._key("text")
|
||||
assert key.startswith(f"{DEDUP_KEY_PREFIX}sha256:")
|
||||
# 后缀是 64 位 sha256 hex
|
||||
suffix = key[len(f"{DEDUP_KEY_PREFIX}sha256:"):]
|
||||
assert len(suffix) == 64
|
||||
|
||||
async def test_lookup_miss_when_empty(self):
|
||||
s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100)
|
||||
assert await s.lookup("text") is None
|
||||
|
||||
async def test_record_then_lookup_hit(self):
|
||||
redis = FakeRedis()
|
||||
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
|
||||
await s.record("text", {"document_id": "doc-1"})
|
||||
hit = await s.lookup("text")
|
||||
assert hit is not None
|
||||
assert hit["document_id"] == "doc-1"
|
||||
|
||||
async def test_record_uses_ttl(self):
|
||||
redis = FakeRedis()
|
||||
s = Sha256DedupStrategy(redis=redis, ttl_seconds=42)
|
||||
await s.record("text", {"x": 1})
|
||||
# 检查写入 Redis 时使用的 TTL
|
||||
key = s._key("text")
|
||||
writes = [(k, v, ttl) for k, v, ttl in redis.writes if k == key]
|
||||
assert writes
|
||||
assert writes[0][2] == 42
|
||||
|
||||
async def test_lookup_redis_error_returns_none(self):
|
||||
"""Redis 抛错时降级为未命中"""
|
||||
redis = AsyncMock()
|
||||
redis.get_json = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
|
||||
assert await s.lookup("text") is None
|
||||
|
||||
async def test_record_redis_error_swallows(self):
|
||||
"""Redis 写入抛错时不向上抛"""
|
||||
redis = AsyncMock()
|
||||
redis.set_json = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
|
||||
await s.record("text", {"x": 1}) # 不抛
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# SimhashDedupStrategy
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestSimhashStrategy:
|
||||
async def test_lookup_miss_when_empty_index(self):
|
||||
s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=3)
|
||||
assert await s.lookup("text") is None
|
||||
|
||||
async def test_record_then_lookup_identical_text(self):
|
||||
"""相同文本 simhash 相同,距离 0 <= 阈值,命中"""
|
||||
redis = FakeRedis()
|
||||
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
||||
await s.record("一段文本", {"document_id": "d1"})
|
||||
hit = await s.lookup("一段文本")
|
||||
assert hit is not None
|
||||
assert hit["document_id"] == "d1"
|
||||
|
||||
async def test_lookup_similar_text_within_threshold(self):
|
||||
"""相似文本(海明距离 <= 阈值)也命中"""
|
||||
redis = FakeRedis()
|
||||
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=64) # 阈值放大确保命中
|
||||
await s.record("原文档文本内容示例", {"document_id": "d1"})
|
||||
# 改一个字符
|
||||
hit = await s.lookup("原文档文本内容示例改")
|
||||
assert hit is not None
|
||||
|
||||
async def test_lookup_different_text_below_threshold_misses(self):
|
||||
"""完全不同文本距离大,未命中"""
|
||||
redis = FakeRedis()
|
||||
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
||||
await s.record("完全不同的第一种文本内容用于测试", {"document_id": "d1"})
|
||||
hit = await s.lookup("另一段毫不相关的内容用于测试去重逻辑")
|
||||
# 距离应该比较大;若碰巧小于阈值(小概率),改大文本差异
|
||||
if hit is not None:
|
||||
# 极小概率命中,放宽断言:至少 record 已写入
|
||||
assert "document_id" in hit
|
||||
else:
|
||||
assert hit is None
|
||||
|
||||
async def test_record_updates_index(self):
|
||||
"""record 把新 simhash 加入索引"""
|
||||
redis = FakeRedis()
|
||||
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
||||
await s.record("文本A", {"x": 1})
|
||||
await s.record("文本B完全不同", {"x": 2})
|
||||
index = await redis.get_json(SimhashDedupStrategy.INDEX_KEY)
|
||||
assert index is not None
|
||||
assert len(index["entries"]) == 2
|
||||
|
||||
async def test_threshold_clamped(self):
|
||||
"""threshold 超出 0~64 范围被夹紧"""
|
||||
s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=999)
|
||||
assert s._threshold == 64
|
||||
s2 = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=-5)
|
||||
assert s2._threshold == 0
|
||||
|
||||
async def test_lookup_redis_error_returns_none(self):
|
||||
redis = AsyncMock()
|
||||
redis.get_json = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
||||
assert await s.lookup("text") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestFactory:
|
||||
def test_none_strategy_when_redis_none(self, isolated_settings_path):
|
||||
"""redis=None 时无论配置如何,都返回 NoopDedupStrategy"""
|
||||
s = get_dedup_strategy(None)
|
||||
assert isinstance(s, NoopDedupStrategy)
|
||||
|
||||
def test_none_strategy_when_config_none(self, isolated_settings_path):
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
|
||||
s = get_dedup_strategy(FakeRedis())
|
||||
assert isinstance(s, NoopDedupStrategy)
|
||||
|
||||
def test_sha256_strategy(self, isolated_settings_path):
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
||||
s = get_dedup_strategy(FakeRedis())
|
||||
assert isinstance(s, Sha256DedupStrategy)
|
||||
|
||||
def test_simhash_strategy(self, isolated_settings_path):
|
||||
rs.update_runtime_settings(
|
||||
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
|
||||
)
|
||||
s = get_dedup_strategy(FakeRedis())
|
||||
assert isinstance(s, SimhashDedupStrategy)
|
||||
assert s._threshold == 5
|
||||
|
||||
def test_cache_same_signature_returns_same_instance(self, isolated_settings_path):
|
||||
"""配置签名相同时复用单例"""
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
||||
s1 = get_dedup_strategy(FakeRedis())
|
||||
s2 = get_dedup_strategy(FakeRedis())
|
||||
assert s1 is s2
|
||||
|
||||
def test_cache_invalidated_on_signature_change(self, isolated_settings_path):
|
||||
"""配置签名变化时重建单例"""
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
||||
s1 = get_dedup_strategy(FakeRedis())
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "simhash"}})
|
||||
s2 = get_dedup_strategy(FakeRedis())
|
||||
assert s1 is not s2
|
||||
assert isinstance(s2, SimhashDedupStrategy)
|
||||
|
||||
def test_invalidate_cache_clears(self, isolated_settings_path):
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
||||
s1 = get_dedup_strategy(FakeRedis())
|
||||
invalidate_dedup_strategy_cache()
|
||||
s2 = get_dedup_strategy(FakeRedis())
|
||||
assert s1 is not s2
|
||||
|
||||
def test_ttl_change_rebuilds(self, isolated_settings_path):
|
||||
"""ttl 变化也触发重建(签名包含 ttl)"""
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 100}})
|
||||
s1 = get_dedup_strategy(FakeRedis())
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 200}})
|
||||
s2 = get_dedup_strategy(FakeRedis())
|
||||
assert s1 is not s2
|
||||
@@ -266,9 +266,15 @@ def test_upload_rejects_empty_text_after_parse(
|
||||
def test_upload_rejects_corrupted_pdf(
|
||||
client: TestClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""损坏的 PDF:code=1001,message 含'文件解析失败',未提交任务"""
|
||||
"""损坏的 PDF:code=1001,message 含'文件解析失败'或'无法从文件提取文本',未提交任务
|
||||
|
||||
file_parser 插件化后:pypdf 失败被捕获并降级到 OCR;若 OCR 不可用/关闭,
|
||||
最终返回空文本,由 upload 端点统一报 '无法从文件提取文本'。
|
||||
关闭 OCR 避免触发 rapidocr 模型下载拖慢测试。
|
||||
"""
|
||||
manager = FakeManager()
|
||||
_inject_manager(monkeypatch, manager)
|
||||
monkeypatch.setattr(document_module.settings, "pdf_ocr_enabled", False)
|
||||
|
||||
resp = client.post(
|
||||
"/api/v1/documents/upload",
|
||||
@@ -277,7 +283,7 @@ def test_upload_rejects_corrupted_pdf(
|
||||
|
||||
body = resp.json()
|
||||
assert body["code"] == 1001
|
||||
assert "文件解析失败" in body["message"]
|
||||
assert "文件解析失败" in body["message"] or "无法从文件提取文本" in body["message"]
|
||||
assert manager.submitted == []
|
||||
|
||||
|
||||
|
||||
+29
-11
@@ -111,10 +111,17 @@ def test_parse_file_no_extension_raises() -> None:
|
||||
parse_file("noext", b"text")
|
||||
|
||||
|
||||
def test_parse_file_corrupted_pdf_raises() -> None:
|
||||
"""损坏的 PDF 抛 ValueError,message 含'文件解析失败'"""
|
||||
with pytest.raises(ValueError, match=r"文件解析失败"):
|
||||
parse_file("bad.pdf", b"not a real pdf")
|
||||
def test_parse_file_corrupted_pdf_returns_empty(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""损坏的 PDF:插件化后文本层提取失败被捕获,OCR 关闭时返回空字符串
|
||||
|
||||
原 test_parse_file_corrupted_pdf_raises 期望 ValueError,但插件化重构后
|
||||
file_parser 设计为优雅降级(pypdf 失败 → OCR 兜底 → 都失败返回空),
|
||||
不再向上抛异常。关闭 OCR 避免触发 rapidocr 模型下载拖慢测试。
|
||||
"""
|
||||
monkeypatch.setattr(settings, "pdf_ocr_enabled", False)
|
||||
assert parse_file("bad.pdf", b"not a real pdf") == ""
|
||||
|
||||
|
||||
def test_parse_file_empty_html_returns_empty_string() -> None:
|
||||
@@ -152,12 +159,23 @@ def test_supported_extensions_contains_expected_set() -> None:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_ocr_state() -> Any:
|
||||
"""每个 OCR 测试前后重置模块级 OCR 引擎状态,避免相互污染"""
|
||||
saved_engine = fp_module._ocr_engine
|
||||
saved_unavailable = fp_module._ocr_unavailable
|
||||
"""每个 OCR 测试前后重置 RapidocrOcrEngine/TesseractOcrEngine 类级状态与插件缓存
|
||||
|
||||
file_parser 插件化后,模块级 _ocr_engine/_ocr_unavailable 已移除,
|
||||
RapidocrOcrEngine 用类级字段 _engine/_unavailable 单例化。
|
||||
测试前重置为干净状态(避免上个测试残留),测试后清理插件缓存。
|
||||
"""
|
||||
# 测试前:重置为干净状态
|
||||
fp_module.RapidocrOcrEngine._engine = None
|
||||
fp_module.RapidocrOcrEngine._unavailable = False
|
||||
fp_module.TesseractOcrEngine._unavailable = False
|
||||
fp_module._ocr_plugin_cache.clear()
|
||||
yield
|
||||
fp_module._ocr_engine = saved_engine
|
||||
fp_module._ocr_unavailable = saved_unavailable
|
||||
# 测试后:再次清理,避免污染后续非 OCR 测试
|
||||
fp_module.RapidocrOcrEngine._engine = None
|
||||
fp_module.RapidocrOcrEngine._unavailable = False
|
||||
fp_module.TesseractOcrEngine._unavailable = False
|
||||
fp_module._ocr_plugin_cache.clear()
|
||||
|
||||
|
||||
class _FakeTextPage:
|
||||
@@ -281,7 +299,7 @@ def test_parse_pdf_ocr_returns_empty_when_dependency_unavailable(
|
||||
monkeypatch.setattr(settings, "pdf_ocr_enabled", True)
|
||||
|
||||
assert parse_file("scan.pdf", b"fake pdf bytes") == ""
|
||||
assert fp_module._ocr_unavailable is True
|
||||
assert fp_module.RapidocrOcrEngine._unavailable is True
|
||||
|
||||
|
||||
def test_parse_pdf_ocr_runtime_exception_falls_back_to_empty(
|
||||
@@ -324,4 +342,4 @@ def test_parse_pdf_text_layer_present_skips_ocr(
|
||||
|
||||
result = parse_file("text.pdf", b"fake pdf bytes")
|
||||
assert result == "这是文本层的内容"
|
||||
assert fp_module._ocr_engine is None
|
||||
assert fp_module.RapidocrOcrEngine._engine is None
|
||||
|
||||
@@ -227,7 +227,11 @@ async def test_get_falls_back_to_redis_then_none() -> None:
|
||||
|
||||
|
||||
def _text_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
"""构造与 Sha256DedupStrategy 一致的 dedup key
|
||||
|
||||
dedup 模块化后 key 形如 dedup:sha256:<sha256hex>(前缀 + 策略名 + hash)。
|
||||
"""
|
||||
return f"sha256:{hashlib.sha256(text.encode('utf-8')).hexdigest()}"
|
||||
|
||||
|
||||
async def test_dedup_hit_reuses_old_doc_id_and_skips_pipeline() -> None:
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
"""LLM 工厂与客户端单元测试:mock httpx 验证 Ollama / OpenAICompatible 两条路径 + 缓存
|
||||
|
||||
通过 monkeypatch 替换 `httpx.AsyncClient` 为伪造的上下文管理器,避免真实网络请求。
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from app.core import runtime_settings as rs
|
||||
from app.services import llm as llm_mod
|
||||
from app.services.llm import (
|
||||
LLMClient,
|
||||
OllamaLLMClient,
|
||||
OpenAICompatibleLLMClient,
|
||||
create_llm_client,
|
||||
invalidate_llm_client_cache,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# httpx 伪造工具
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, status_code: int = 200, json_data: dict | None = None) -> None:
|
||||
self.status_code = status_code
|
||||
self._json = json_data or {}
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status_code >= 400:
|
||||
raise httpx.HTTPStatusError(
|
||||
f"HTTP {self.status_code}", request=httpx.Request("POST", "http://x"), response=self
|
||||
)
|
||||
|
||||
def json(self) -> dict:
|
||||
return self._json
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
"""伪造 httpx.AsyncClient 上下文管理器
|
||||
|
||||
用法:把 _FakeAsyncClient.next_response 设为期望响应,构造后 post/get 返回该响应。
|
||||
每次 __init__ 记录构造参数到 _FakeAsyncClient.last_kwargs。
|
||||
"""
|
||||
|
||||
next_response: _FakeResponse | None = None
|
||||
last_kwargs: dict | None = None
|
||||
last_instances: list["_FakeAsyncClient"] = []
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.post = AsyncMock()
|
||||
self.get = AsyncMock()
|
||||
if _FakeAsyncClient.next_response is not None:
|
||||
self.post.return_value = _FakeAsyncClient.next_response
|
||||
self.get.return_value = _FakeAsyncClient.next_response
|
||||
_FakeAsyncClient.last_kwargs = kwargs
|
||||
_FakeAsyncClient.last_instances.append(self)
|
||||
|
||||
async def __aenter__(self) -> "_FakeAsyncClient":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def reset(cls) -> None:
|
||||
cls.next_response = None
|
||||
cls.last_kwargs = None
|
||||
cls.last_instances = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patch_httpx(monkeypatch: pytest.MonkeyPatch):
|
||||
"""替换 httpx.AsyncClient 为伪造客户端"""
|
||||
_FakeAsyncClient.reset()
|
||||
monkeypatch.setattr(llm_mod.httpx, "AsyncClient", _FakeAsyncClient)
|
||||
# ollama 模块内 OllamaClient 也用 httpx.AsyncClient
|
||||
from app.services import ollama as ollama_mod
|
||||
monkeypatch.setattr(ollama_mod.httpx, "AsyncClient", _FakeAsyncClient)
|
||||
yield _FakeAsyncClient
|
||||
_FakeAsyncClient.reset()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_settings_path(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
path = tmp_path / "runtime_settings.json"
|
||||
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
|
||||
rs._runtime_settings = None
|
||||
invalidate_llm_client_cache()
|
||||
yield path
|
||||
rs._runtime_settings = None
|
||||
invalidate_llm_client_cache()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# OllamaLLMClient
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestOllamaLLMClient:
|
||||
async def test_generate_returns_response_field(self, patch_httpx):
|
||||
"""Ollama 响应:取 data['response'] 字段"""
|
||||
patch_httpx.next_response = _FakeResponse(
|
||||
json_data={"response": "你好世界", "model": "qwen2.5:1.5b"}
|
||||
)
|
||||
client = OllamaLLMClient(base_url="http://ollama:11434", model="qwen2.5:1.5b")
|
||||
out = await client.generate("prompt")
|
||||
assert out == "你好世界"
|
||||
# 校验请求 url 与 payload
|
||||
fake = patch_httpx.last_instances[-1]
|
||||
fake.post.assert_awaited_once()
|
||||
call_args = fake.post.await_args
|
||||
assert call_args.args[0] == "http://ollama:11434/api/generate"
|
||||
payload = call_args.kwargs["json"]
|
||||
assert payload["model"] == "qwen2.5:1.5b"
|
||||
assert payload["prompt"] == "prompt"
|
||||
assert payload["stream"] is False
|
||||
|
||||
async def test_generate_json_mode_adds_format(self, patch_httpx):
|
||||
patch_httpx.next_response = _FakeResponse(json_data={"response": "{}"})
|
||||
client = OllamaLLMClient(base_url="http://o:11434", model="m")
|
||||
await client.generate("p", json_mode=True)
|
||||
payload = patch_httpx.last_instances[-1].post.await_args.kwargs["json"]
|
||||
assert payload["format"] == "json"
|
||||
|
||||
async def test_is_available_true(self, patch_httpx):
|
||||
patch_httpx.next_response = _FakeResponse(status_code=200)
|
||||
client = OllamaLLMClient(base_url="http://o:11434", model="m")
|
||||
assert await client.is_available() is True
|
||||
|
||||
async def test_is_available_false_on_http_error(self, monkeypatch):
|
||||
"""httpx 抛 HTTPError 时 OllamaClient 内部捕获返回 False,OllamaLLMClient 透传"""
|
||||
|
||||
class _RaisingClient:
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
return False
|
||||
|
||||
async def get(self, *args, **kwargs):
|
||||
raise httpx.HTTPError("conn refused")
|
||||
|
||||
from app.services import ollama as ollama_mod
|
||||
|
||||
monkeypatch.setattr(ollama_mod.httpx, "AsyncClient", lambda *a, **kw: _RaisingClient())
|
||||
client = OllamaLLMClient(base_url="http://o:11434", model="m")
|
||||
assert await client.is_available() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# OpenAICompatibleLLMClient
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestOpenAICompatibleClient:
|
||||
async def test_generate_returns_choices_content(self, patch_httpx):
|
||||
patch_httpx.next_response = _FakeResponse(
|
||||
json_data={"choices": [{"message": {"content": "答案"}}]}
|
||||
)
|
||||
client = OpenAICompatibleLLMClient(
|
||||
base_url="https://api.openai.com/v1", api_key="sk-x", model="gpt-4o-mini"
|
||||
)
|
||||
out = await client.generate("hello")
|
||||
assert out == "答案"
|
||||
fake = patch_httpx.last_instances[-1]
|
||||
call_args = fake.post.await_args
|
||||
assert call_args.args[0] == "https://api.openai.com/v1/chat/completions"
|
||||
payload = call_args.kwargs["json"]
|
||||
assert payload["model"] == "gpt-4o-mini"
|
||||
assert payload["messages"] == [{"role": "user", "content": "hello"}]
|
||||
assert payload["stream"] is False
|
||||
# Authorization header
|
||||
headers = call_args.kwargs["headers"]
|
||||
assert headers["Authorization"] == "Bearer sk-x"
|
||||
|
||||
async def test_generate_json_mode_adds_response_format(self, patch_httpx):
|
||||
patch_httpx.next_response = _FakeResponse(
|
||||
json_data={"choices": [{"message": {"content": "{}"}}]}
|
||||
)
|
||||
client = OpenAICompatibleLLMClient(
|
||||
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
|
||||
)
|
||||
await client.generate("p", json_mode=True)
|
||||
payload = patch_httpx.last_instances[-1].post.await_args.kwargs["json"]
|
||||
assert payload["response_format"] == {"type": "json_object"}
|
||||
|
||||
async def test_generate_empty_choices_returns_empty(self, patch_httpx):
|
||||
patch_httpx.next_response = _FakeResponse(json_data={"choices": []})
|
||||
client = OpenAICompatibleLLMClient(
|
||||
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
|
||||
)
|
||||
assert await client.generate("p") == ""
|
||||
|
||||
async def test_is_available_true(self, patch_httpx):
|
||||
patch_httpx.next_response = _FakeResponse(status_code=200)
|
||||
client = OpenAICompatibleLLMClient(
|
||||
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
|
||||
)
|
||||
assert await client.is_available() is True
|
||||
|
||||
async def test_is_available_false_on_http_error(self, monkeypatch):
|
||||
"""httpx 抛 HTTPError 时返回 False"""
|
||||
client = OpenAICompatibleLLMClient(
|
||||
base_url="https://api.openai.com/v1", api_key="sk-x", model="m"
|
||||
)
|
||||
|
||||
class _Raiser:
|
||||
async def __aenter__(self):
|
||||
raise httpx.HTTPError("conn refused")
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(llm_mod.httpx, "AsyncClient", lambda *a, **kw: _Raiser())
|
||||
assert await client.is_available() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestFactory:
|
||||
def test_ollama_provider_returns_ollama_client(self, isolated_settings_path):
|
||||
"""runtime_settings 默认 provider=ollama,工厂返回 OllamaLLMClient"""
|
||||
# 确保配置走 ollama(默认即 ollama)
|
||||
client = create_llm_client("summarize", use_cache=False)
|
||||
assert isinstance(client, OllamaLLMClient)
|
||||
|
||||
def test_openai_compatible_provider_returns_openai_client(
|
||||
self, isolated_settings_path
|
||||
):
|
||||
rs.update_runtime_settings(
|
||||
{"models": {"query": {"provider": "openai_compatible", "api_key": "sk-x", "model": "gpt-4o-mini"}}}
|
||||
)
|
||||
client = create_llm_client("query", use_cache=False)
|
||||
assert isinstance(client, OpenAICompatibleLLMClient)
|
||||
assert client.model == "gpt-4o-mini"
|
||||
assert client.api_key == "sk-x"
|
||||
|
||||
def test_openai_compatible_falls_back_to_settings(self, isolated_settings_path):
|
||||
"""openai_compatible 时 base_url/api_key 为空用 settings 默认;model 为空用 gpt-4o-mini"""
|
||||
# 显式置空 model 以触发工厂默认值(env fallback 会预填 ollama_model)
|
||||
rs.update_runtime_settings(
|
||||
{"models": {"classify": {"provider": "openai_compatible", "model": ""}}}
|
||||
)
|
||||
from app.config import settings
|
||||
|
||||
client = create_llm_client("classify", use_cache=False)
|
||||
assert isinstance(client, OpenAICompatibleLLMClient)
|
||||
assert client.api_key == settings.openai_api_key
|
||||
assert client.model == "gpt-4o-mini" # openai_compatible 默认模型
|
||||
|
||||
def test_cache_returns_same_instance(self, isolated_settings_path):
|
||||
"""use_cache=True 时同 purpose 复用单例"""
|
||||
c1 = create_llm_client("summarize")
|
||||
c2 = create_llm_client("summarize")
|
||||
assert c1 is c2
|
||||
|
||||
def test_cache_different_purposes_different_instances(self, isolated_settings_path):
|
||||
c1 = create_llm_client("summarize")
|
||||
c2 = create_llm_client("query")
|
||||
assert c1 is not c2
|
||||
|
||||
def test_no_cache_returns_new_instance_each_call(self, isolated_settings_path):
|
||||
c1 = create_llm_client("summarize", use_cache=False)
|
||||
c2 = create_llm_client("summarize", use_cache=False)
|
||||
assert c1 is not c2
|
||||
|
||||
def test_invalidate_clears_cache(self, isolated_settings_path):
|
||||
c1 = create_llm_client("summarize")
|
||||
invalidate_llm_client_cache()
|
||||
c2 = create_llm_client("summarize")
|
||||
assert c1 is not c2
|
||||
|
||||
def test_invalidate_single_purpose(self, isolated_settings_path):
|
||||
c_sum = create_llm_client("summarize")
|
||||
c_qry = create_llm_client("query")
|
||||
invalidate_llm_client_cache("summarize")
|
||||
# summarize 已清,query 未清
|
||||
assert create_llm_client("summarize") is not c_sum
|
||||
assert create_llm_client("query") is c_qry
|
||||
|
||||
def test_client_implements_protocol(self, isolated_settings_path):
|
||||
"""OllamaLLMClient 与 OpenAICompatibleLLMClient 都满足 LLMClient Protocol"""
|
||||
c1 = create_llm_client("summarize", use_cache=False)
|
||||
assert isinstance(c1, LLMClient)
|
||||
rs.update_runtime_settings(
|
||||
{"models": {"summarize": {"provider": "openai_compatible", "api_key": "k"}}}
|
||||
)
|
||||
c2 = create_llm_client("summarize", use_cache=False)
|
||||
assert isinstance(c2, LLMClient)
|
||||
@@ -0,0 +1,254 @@
|
||||
"""RuntimeSettings 单元测试:默认值 / 加载 / 保存 / 部分更新 / 重置 / 单例
|
||||
|
||||
每个测试通过 monkeypatch 把 RUNTIME_SETTINGS_PATH 指向独立临时文件,并在前后
|
||||
重置模块级 `_runtime_settings` 单例,避免跨测试串扰。
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core import runtime_settings as rs
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_settings_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
"""每个测试独立持久化路径,并在前后清空模块级单例"""
|
||||
path = tmp_path / "runtime_settings.json"
|
||||
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
|
||||
# 重置单例,强制下次 get_runtime_settings 重新加载
|
||||
rs._runtime_settings = None
|
||||
yield path
|
||||
rs._runtime_settings = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认值
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestDefaults:
|
||||
def test_default_models_provider_is_ollama(self):
|
||||
cfg = rs.RuntimeSettings()
|
||||
assert cfg.models.summarize.provider == "ollama"
|
||||
assert cfg.models.query.provider == "ollama"
|
||||
assert cfg.models.classify.provider == "ollama"
|
||||
|
||||
def test_default_parsers_plugins(self):
|
||||
cfg = rs.RuntimeSettings()
|
||||
assert cfg.parsers.ocr.plugin == "rapidocr"
|
||||
assert cfg.parsers.pdf.plugin == "pypdf"
|
||||
assert cfg.parsers.docx.plugin == "python_docx"
|
||||
|
||||
def test_default_dedup(self):
|
||||
cfg = rs.RuntimeSettings()
|
||||
assert cfg.dedup.strategy == "sha256"
|
||||
assert cfg.dedup.simhash_threshold == 3
|
||||
assert cfg.dedup.ttl_seconds == 86400
|
||||
|
||||
def test_default_with_env_fallback_uses_ollama_env(self):
|
||||
"""无持久化文件时,base_url/model 取自 settings.ollama_*"""
|
||||
cfg = rs._default_with_env_fallback()
|
||||
from app.config import settings
|
||||
|
||||
assert cfg.models.summarize.base_url == settings.ollama_base_url
|
||||
assert cfg.models.summarize.model == settings.ollama_model
|
||||
assert cfg.models.query.base_url == settings.ollama_base_url
|
||||
assert cfg.models.classify.model == settings.ollama_model
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 加载
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestLoad:
|
||||
def test_missing_file_returns_env_fallback(self, isolated_settings_path: Path):
|
||||
"""文件不存在时回退到带 env 兜底的默认值(不抛异常)"""
|
||||
assert not isolated_settings_path.exists()
|
||||
cfg = rs.load_runtime_settings()
|
||||
# base_url 应来自 env fallback
|
||||
from app.config import settings
|
||||
|
||||
assert cfg.models.summarize.base_url == settings.ollama_base_url
|
||||
|
||||
def test_valid_file_parsed(self, isolated_settings_path: Path):
|
||||
isolated_settings_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"models": {
|
||||
"summarize": {"provider": "openai_compatible", "model": "gpt-4o-mini", "api_key": "k"}
|
||||
},
|
||||
"dedup": {"strategy": "simhash", "simhash_threshold": 5},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
cfg = rs.load_runtime_settings()
|
||||
assert cfg.models.summarize.provider == "openai_compatible"
|
||||
assert cfg.models.summarize.model == "gpt-4o-mini"
|
||||
assert cfg.models.summarize.api_key == "k"
|
||||
assert cfg.dedup.strategy == "simhash"
|
||||
assert cfg.dedup.simhash_threshold == 5
|
||||
# 未指定的字段保留默认
|
||||
assert cfg.dedup.ttl_seconds == 86400
|
||||
assert cfg.models.query.provider == "ollama"
|
||||
|
||||
def test_corrupted_file_falls_back(self, isolated_settings_path: Path):
|
||||
isolated_settings_path.write_text("not-json{", encoding="utf-8")
|
||||
cfg = rs.load_runtime_settings()
|
||||
# 回退到默认(ollama provider)
|
||||
assert cfg.models.summarize.provider == "ollama"
|
||||
|
||||
def test_invalid_values_falls_back(self, isolated_settings_path: Path):
|
||||
"""字段值非法(如未知 provider)时整体回退默认"""
|
||||
isolated_settings_path.write_text(
|
||||
json.dumps({"models": {"summarize": {"provider": "unknown_provider"}}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
cfg = rs.load_runtime_settings()
|
||||
assert cfg.models.summarize.provider == "ollama" # 回退默认
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 保存
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestSave:
|
||||
def test_save_writes_valid_json(self, isolated_settings_path: Path):
|
||||
cfg = rs.RuntimeSettings()
|
||||
cfg.dedup.strategy = "simhash"
|
||||
rs.save_runtime_settings(cfg)
|
||||
assert isolated_settings_path.exists()
|
||||
data = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
|
||||
assert data["dedup"]["strategy"] == "simhash"
|
||||
|
||||
def test_save_creates_parent_dir(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
path = tmp_path / "nested" / "deep" / "runtime_settings.json"
|
||||
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
|
||||
rs.save_runtime_settings(rs.RuntimeSettings())
|
||||
assert path.exists()
|
||||
|
||||
def test_save_atomic_no_tmp_left(self, isolated_settings_path: Path):
|
||||
"""保存后同目录无残留 .tmp 临时文件"""
|
||||
rs.save_runtime_settings(rs.RuntimeSettings())
|
||||
tmps = list(isolated_settings_path.parent.glob(".runtime_settings.*.tmp"))
|
||||
assert tmps == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 单例 + reload
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestSingleton:
|
||||
def test_get_returns_singleton(self, isolated_settings_path: Path):
|
||||
cfg1 = rs.get_runtime_settings()
|
||||
cfg2 = rs.get_runtime_settings()
|
||||
assert cfg1 is cfg2
|
||||
|
||||
def test_reload_rereads_disk(self, isolated_settings_path: Path):
|
||||
"""reload 强制重新读盘,单例替换为新对象"""
|
||||
cfg1 = rs.get_runtime_settings()
|
||||
# 直接改盘上文件
|
||||
isolated_settings_path.write_text(
|
||||
json.dumps({"dedup": {"strategy": "none"}}), encoding="utf-8"
|
||||
)
|
||||
cfg2 = rs.reload_runtime_settings()
|
||||
assert cfg2 is not cfg1
|
||||
assert cfg2.dedup.strategy == "none"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 部分更新(深合并)
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestUpdate:
|
||||
def test_partial_update_models_summarize(self, isolated_settings_path: Path):
|
||||
rs.get_runtime_settings() # 初始化单例
|
||||
new_cfg = rs.update_runtime_settings(
|
||||
{"models": {"summarize": {"model": "qwen2.5:3b"}}}
|
||||
)
|
||||
assert new_cfg.models.summarize.model == "qwen2.5:3b"
|
||||
# 其他字段保留
|
||||
assert new_cfg.models.summarize.provider == "ollama"
|
||||
assert new_cfg.models.query.provider == "ollama"
|
||||
|
||||
def test_partial_update_dedup(self, isolated_settings_path: Path):
|
||||
rs.get_runtime_settings()
|
||||
new_cfg = rs.update_runtime_settings(
|
||||
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
|
||||
)
|
||||
assert new_cfg.dedup.strategy == "simhash"
|
||||
assert new_cfg.dedup.simhash_threshold == 5
|
||||
# ttl 未在 patch 中,保留默认
|
||||
assert new_cfg.dedup.ttl_seconds == 86400
|
||||
|
||||
def test_update_persists_to_disk(self, isolated_settings_path: Path):
|
||||
rs.get_runtime_settings()
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
|
||||
data = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
|
||||
assert data["dedup"]["strategy"] == "none"
|
||||
|
||||
def test_update_replaces_singleton(self, isolated_settings_path: Path):
|
||||
old = rs.get_runtime_settings()
|
||||
new = rs.update_runtime_settings({"dedup": {"strategy": "none"}})
|
||||
assert new is not old
|
||||
# 后续 get 拿到的是新单例
|
||||
assert rs.get_runtime_settings() is new
|
||||
|
||||
def test_update_empty_patch_keeps_all(self, isolated_settings_path: Path):
|
||||
"""空 patch 不改变任何字段"""
|
||||
rs.get_runtime_settings()
|
||||
new_cfg = rs.update_runtime_settings({})
|
||||
assert new_cfg.dedup.strategy == "sha256"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 重置
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestReset:
|
||||
def test_reset_returns_defaults(self, isolated_settings_path: Path):
|
||||
# 先污染
|
||||
rs.get_runtime_settings()
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
|
||||
assert rs.get_runtime_settings().dedup.strategy == "none"
|
||||
# 重置
|
||||
cfg = rs.reset_runtime_settings()
|
||||
assert cfg.dedup.strategy == "sha256"
|
||||
assert cfg.dedup.simhash_threshold == 3
|
||||
assert cfg.parsers.ocr.plugin == "rapidocr"
|
||||
|
||||
def test_reset_persists_to_disk(self, isolated_settings_path: Path):
|
||||
rs.get_runtime_settings()
|
||||
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
|
||||
rs.reset_runtime_settings()
|
||||
data = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
|
||||
assert data["dedup"]["strategy"] == "sha256"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 深合并工具函数
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestDeepMerge:
|
||||
def test_nested_dict_merged(self):
|
||||
target = {"a": {"b": 1, "c": 2}, "d": 3}
|
||||
rs._deep_merge(target, {"a": {"b": 10}})
|
||||
assert target == {"a": {"b": 10, "c": 2}, "d": 3}
|
||||
|
||||
def test_non_dict_overrides(self):
|
||||
target = {"a": {"b": 1}}
|
||||
rs._deep_merge(target, {"a": 99})
|
||||
assert target == {"a": 99}
|
||||
|
||||
def test_new_key_added(self):
|
||||
target = {"a": 1}
|
||||
rs._deep_merge(target, {"b": 2})
|
||||
assert target == {"a": 1, "b": 2}
|
||||
Reference in New Issue
Block a user