feat: 完成全量功能开发，包括前端管理后台与后端服务优化
此提交实现了完整的知识库管理系统： 1. 新增Vue3 + Antd Vue前端管理后台，包含登录、文档管理、检索、类目设置等完整页面 2. 重构后端LLM调用抽象层，支持Ollama与OpenAI兼容服务动态切换 3. 调整默认嵌入模型配置为本地bge-m3模式 4. 优化入库任务去重逻辑与缓存清理机制 5. 完善Docker镜像构建与docker-compose部署配置 6. 修复多项测试用例与兼容性问题 7. 新增运行时配置API，支持动态调整系统参数
This commit is contained in:
+346
-87
@@ -1,14 +1,19 @@
|
||||
"""多格式文件文本提取：按扩展名分发到对应解析器
|
||||
"""多格式文件文本提取：按扩展名分发到对应解析器（插件化）
|
||||
|
||||
支持的扩展名：
|
||||
- .txt / .md：UTF-8 解码（errors="replace" 兜底）
|
||||
- .html / .htm：标准库 html.parser 剥离标签提取可见文本
|
||||
- .pdf：pypdf 逐页 extract_text 拼接；文本层为空（扫描件/图片型）时
|
||||
自动降级为 OCR（pypdfium2 渲染 + rapidocr-onnxruntime 识别）
|
||||
- .docx：python-docx 段落文本拼接（不含表格/页眉页脚）
|
||||
- .pdf：PDF 文本层提取插件（默认 pypdf）；文本层为空（扫描件）时降级到 OCR 插件
|
||||
- .docx：DOCX 解析插件（默认 python-docx）
|
||||
|
||||
插件化设计：
|
||||
- PdfTextExtractor / DocxParser / OcrEngine 三个 Protocol
|
||||
- 每种插件类型有注册表 + 默认实现 + 工厂
|
||||
- 工厂根据 runtime_settings.parsers 选择具体插件
|
||||
- 未注册或依赖缺失时降级到默认插件并告警
|
||||
|
||||
未识别扩展名抛 ValueError("不支持的文件类型: {ext}")；
|
||||
解析异常统一包装为 ValueError("文件解析失败: {detail}")，原异常链式保留。
|
||||
解析异常统一包装为 ValueError("文件解析失败: {detail}")。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,17 +22,19 @@ import io
|
||||
from collections.abc import Callable
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
import structlog
|
||||
|
||||
from app.config import settings
|
||||
from app.core.runtime_settings import get_runtime_settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# 模块级懒加载 OCR 引擎单例（首次调用时初始化，避免无扫描件场景白白下载模型）
|
||||
_ocr_engine: Any | None = None
|
||||
_ocr_unavailable: bool = False # 标记 OCR 依赖不可用，后续直接跳过避免重复尝试
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# HTML / Text 解析（无插件化需求，保留原实现）
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _VisibleTextExtractor(HTMLParser):
|
||||
@@ -51,13 +58,11 @@ class _VisibleTextExtractor(HTMLParser):
|
||||
self._parts.append(data)
|
||||
|
||||
def get_text(self) -> str:
|
||||
# 块级标签间用空格连接，再折叠多余空白
|
||||
text = " ".join(self._parts)
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def _decode_html(content: bytes) -> str:
|
||||
"""HTML 内容解码：优先 utf-8（带 BOM），失败回退 latin-1"""
|
||||
try:
|
||||
return content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
@@ -65,108 +70,362 @@ def _decode_html(content: bytes) -> str:
|
||||
|
||||
|
||||
def _parse_text(content: bytes) -> str:
|
||||
"""UTF-8 解码（errors=replace 兜底），保留原字符"""
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _parse_html(content: bytes) -> str:
|
||||
"""HTML 剥离标签，保留可见文本"""
|
||||
parser = _VisibleTextExtractor()
|
||||
parser.feed(_decode_html(content))
|
||||
parser.close()
|
||||
return parser.get_text()
|
||||
|
||||
|
||||
def _parse_pdf(content: bytes) -> str:
|
||||
"""PDF 解析：优先 pypdf extract_text；文本层为空（扫描件）时降级 OCR
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 插件协议
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
OCR 流程：pypdfium2 渲染每页为 PIL Image → rapidocr-onnxruntime 识别 →
|
||||
拼接每页识别出的文本。受 settings.pdf_ocr_* 控制：开关、最大页数、DPI。
|
||||
OCR 依赖未安装或运行异常时降级返回空字符串（由上游 upload 端点拒绝入库）。
|
||||
|
||||
@runtime_checkable
|
||||
class PdfTextExtractor(Protocol):
|
||||
"""PDF 文本层提取插件协议"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
"""从 PDF 二进制内容提取文本层；无文本层返回空字符串"""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DocxParser(Protocol):
|
||||
"""DOCX 解析插件协议"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
"""从 DOCX 二进制内容提取段落文本"""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class OcrEngine(Protocol):
|
||||
"""OCR 引擎插件协议"""
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
"""对 PDF 跑 OCR，返回识别文本；失败返回空字符串"""
|
||||
...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 PDF 插件：pypdf
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PypdfTextExtractor:
|
||||
"""pypdf 文本层提取（默认）"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(io.BytesIO(content))
|
||||
parts: list[str] = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 备选 PDF 插件：pdfplumber（lazy import，依赖缺失时不可用）
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PdfplumberTextExtractor:
|
||||
"""pdfplumber 文本层提取（备选，对复杂排版更友好）"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
import pdfplumber # type: ignore[import-untyped]
|
||||
|
||||
parts: list[str] = []
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
for page in pdf.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 DOCX 插件：python-docx
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class PythonDocxParser:
|
||||
"""python-docx 段落提取（默认）"""
|
||||
|
||||
def extract_text(self, content: bytes) -> str:
|
||||
from docx import Document # type: ignore[import-untyped]
|
||||
|
||||
document = Document(io.BytesIO(content))
|
||||
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 默认 OCR 插件：rapidocr-onnxruntime + pypdfium2
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class RapidocrOcrEngine:
|
||||
"""rapidocr-onnxruntime OCR 引擎（默认）
|
||||
|
||||
流程：pypdfium2 渲染每页为 PIL Image → rapidocr 识别 → 拼接。
|
||||
类级懒加载单例（_engine/_unavailable 为类属性，跨实例共享），
|
||||
依赖缺失时降级返回空文本。
|
||||
"""
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(io.BytesIO(content))
|
||||
parts: list[str] = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
parts.append(text)
|
||||
text_layer = "\n".join(parts).strip()
|
||||
_engine: Any | None = None
|
||||
_unavailable: bool = False
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
if self._unavailable:
|
||||
return ""
|
||||
|
||||
# 1. 懒加载 OCR 引擎（类级单例，跨实例共享）
|
||||
if self._engine is None:
|
||||
try:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
RapidocrOcrEngine._engine = RapidOCR()
|
||||
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
|
||||
except Exception:
|
||||
RapidocrOcrEngine._unavailable = True
|
||||
logger.warning(
|
||||
"OCR 依赖不可用，扫描件 PDF 将无法提取文本", exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
# 2. 渲染并识别
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
result, _ = self._engine(pil_image)
|
||||
if result:
|
||||
lines = [
|
||||
item[1]
|
||||
for item in result
|
||||
if item and len(item) >= 2 and item[1]
|
||||
]
|
||||
if lines:
|
||||
page_texts.append("\n".join(lines))
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"PDF OCR 失败，降级返回空文本", error=str(exc), exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 备选 OCR 插件：tesseract（lazy import，依赖缺失时不可用）
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TesseractOcrEngine:
|
||||
"""pytesseract + pdfium OCR（备选）
|
||||
|
||||
需要系统安装 tesseract 二进制与语言包。
|
||||
类级 _unavailable 单例，跨实例共享。
|
||||
"""
|
||||
|
||||
_unavailable: bool = False
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
if self._unavailable:
|
||||
return ""
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
import pytesseract # type: ignore[import-untyped]
|
||||
from PIL import Image # type: ignore[import-untyped]
|
||||
except Exception:
|
||||
TesseractOcrEngine._unavailable = True
|
||||
logger.warning("tesseract 依赖不可用，降级返回空文本", exc_info=True)
|
||||
return ""
|
||||
|
||||
try:
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
text = pytesseract.image_to_string(pil_image, lang="chi_sim+eng")
|
||||
if text:
|
||||
page_texts.append(text)
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"tesseract OCR 失败，降级返回空文本", error=str(exc), exc_info=True
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
class NoopOcrEngine:
|
||||
"""关闭 OCR 的占位插件（strategy=none 时使用）"""
|
||||
|
||||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 插件注册表
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_PDF_PLUGINS: dict[str, type[PdfTextExtractor]] = {
|
||||
"pypdf": PypdfTextExtractor,
|
||||
"pdfplumber": PdfplumberTextExtractor,
|
||||
}
|
||||
|
||||
_DOCX_PLUGINS: dict[str, type[DocxParser]] = {
|
||||
"python_docx": PythonDocxParser,
|
||||
}
|
||||
|
||||
_OCR_PLUGINS: dict[str, type[OcrEngine]] = {
|
||||
"rapidocr": RapidocrOcrEngine,
|
||||
"tesseract": TesseractOcrEngine,
|
||||
"none": NoopOcrEngine,
|
||||
}
|
||||
|
||||
|
||||
def list_pdf_plugins() -> list[str]:
|
||||
return list(_PDF_PLUGINS.keys())
|
||||
|
||||
|
||||
def list_docx_plugins() -> list[str]:
|
||||
return list(_DOCX_PLUGINS.keys())
|
||||
|
||||
|
||||
def list_ocr_plugins() -> list[str]:
|
||||
return list(_OCR_PLUGINS.keys())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
# 进程级插件单例缓存
|
||||
_pdf_plugin_cache: dict[str, PdfTextExtractor] = {}
|
||||
_docx_plugin_cache: dict[str, DocxParser] = {}
|
||||
_ocr_plugin_cache: dict[str, OcrEngine] = {}
|
||||
|
||||
|
||||
def _get_pdf_plugin() -> PdfTextExtractor:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.pdf.plugin or "pypdf"
|
||||
if name in _pdf_plugin_cache:
|
||||
return _pdf_plugin_cache[name]
|
||||
cls = _PDF_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 PDF 插件，降级到 pypdf", requested=name)
|
||||
cls = PypdfTextExtractor
|
||||
name = "pypdf"
|
||||
instance = cls()
|
||||
_pdf_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def _get_docx_plugin() -> DocxParser:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.docx.plugin or "python_docx"
|
||||
if name in _docx_plugin_cache:
|
||||
return _docx_plugin_cache[name]
|
||||
cls = _DOCX_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 DOCX 插件，降级到 python_docx", requested=name)
|
||||
cls = PythonDocxParser
|
||||
name = "python_docx"
|
||||
instance = cls()
|
||||
_docx_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def _get_ocr_plugin() -> OcrEngine:
|
||||
rt = get_runtime_settings()
|
||||
name = rt.parsers.ocr.plugin or "rapidocr"
|
||||
if name in _ocr_plugin_cache:
|
||||
return _ocr_plugin_cache[name]
|
||||
cls = _OCR_PLUGINS.get(name)
|
||||
if cls is None:
|
||||
logger.warning("未知 OCR 插件，降级到 rapidocr", requested=name)
|
||||
cls = RapidocrOcrEngine
|
||||
name = "rapidocr"
|
||||
instance = cls()
|
||||
_ocr_plugin_cache[name] = instance
|
||||
return instance
|
||||
|
||||
|
||||
def invalidate_parser_plugin_cache(kind: str | None = None) -> None:
|
||||
"""清除插件缓存（runtime_settings 更新后调用）"""
|
||||
if kind is None or kind == "pdf":
|
||||
_pdf_plugin_cache.clear()
|
||||
if kind is None or kind == "docx":
|
||||
_docx_plugin_cache.clear()
|
||||
if kind is None or kind == "ocr":
|
||||
_ocr_plugin_cache.clear()
|
||||
# 重置 RapidocrOcrEngine 类级标记，允许重新初始化
|
||||
RapidocrOcrEngine._engine = None
|
||||
RapidocrOcrEngine._unavailable = False
|
||||
TesseractOcrEngine._unavailable = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 顶层解析函数
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _parse_pdf(content: bytes) -> str:
|
||||
"""PDF 解析：优先文本层插件；文本层为空时降级 OCR 插件"""
|
||||
rt = get_runtime_settings()
|
||||
pdf_plugin = _get_pdf_plugin()
|
||||
try:
|
||||
text_layer = pdf_plugin.extract_text(content)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"PDF 文本层提取失败，尝试 OCR 降级",
|
||||
plugin=rt.parsers.pdf.plugin,
|
||||
error=str(exc),
|
||||
exc_info=True,
|
||||
)
|
||||
text_layer = ""
|
||||
|
||||
# 文本层非空：直接返回
|
||||
if text_layer:
|
||||
return text_layer
|
||||
|
||||
# 文本层为空 → 尝试 OCR 降级
|
||||
# 文本层为空 → OCR 降级
|
||||
if not settings.pdf_ocr_enabled:
|
||||
return ""
|
||||
|
||||
ocr_text = _ocr_pdf(
|
||||
content, max_pages=settings.pdf_ocr_max_pages, dpi=settings.pdf_ocr_dpi
|
||||
ocr_plugin = _get_ocr_plugin()
|
||||
return ocr_plugin.ocr_pdf(
|
||||
content,
|
||||
max_pages=rt.parsers.ocr.params.get("max_pages", settings.pdf_ocr_max_pages),
|
||||
dpi=rt.parsers.ocr.params.get("dpi", settings.pdf_ocr_dpi),
|
||||
)
|
||||
return ocr_text
|
||||
|
||||
|
||||
def _ocr_pdf(content: bytes, max_pages: int, dpi: int) -> str:
|
||||
"""对扫描件 PDF 跑 OCR：渲染每页 → 识别 → 拼接
|
||||
|
||||
返回空字符串的场景：依赖未安装 / 渲染或识别异常 / 无识别结果。
|
||||
任何异常仅告警不抛出，由上游按"无法提取文本"处理。
|
||||
"""
|
||||
global _ocr_engine, _ocr_unavailable
|
||||
|
||||
if _ocr_unavailable:
|
||||
return ""
|
||||
|
||||
# 1. 懒加载 OCR 引擎
|
||||
if _ocr_engine is None:
|
||||
try:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
|
||||
_ocr_engine = RapidOCR()
|
||||
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
|
||||
except Exception:
|
||||
_ocr_unavailable = True
|
||||
logger.warning("OCR 依赖不可用，扫描件 PDF 将无法提取文本", exc_info=True)
|
||||
return ""
|
||||
|
||||
# 2. 渲染并识别
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
scale = max(1.0, dpi / 72.0)
|
||||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||||
total = min(len(pdf), max(1, max_pages))
|
||||
page_texts: list[str] = []
|
||||
for i in range(total):
|
||||
page = pdf[i]
|
||||
pil_image = page.render(scale=scale).to_pil()
|
||||
result, _ = _ocr_engine(pil_image)
|
||||
if result:
|
||||
# result: [[box, text, score], ...]，按行拼接
|
||||
lines = [
|
||||
item[1] for item in result if item and len(item) >= 2 and item[1]
|
||||
]
|
||||
if lines:
|
||||
page_texts.append("\n".join(lines))
|
||||
pdf.close()
|
||||
return "\n".join(page_texts).strip()
|
||||
except Exception as exc:
|
||||
logger.warning("PDF OCR 失败，降级返回空文本", error=str(exc), exc_info=True)
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_docx(content: bytes) -> str:
|
||||
"""DOCX 段落文本拼接（不含表格/页眉页脚）"""
|
||||
from docx import Document # type: ignore[import-untyped]
|
||||
|
||||
document = Document(io.BytesIO(content))
|
||||
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
|
||||
return "\n".join(parts).strip()
|
||||
"""DOCX 解析"""
|
||||
return _get_docx_plugin().extract_text(content)
|
||||
|
||||
|
||||
# 扩展名 → 解析函数映射（启动时构建，避免每次请求重复构造）
|
||||
# 扩展名 → 解析函数映射
|
||||
_PARSERS: dict[str, Callable[[bytes], str]] = {
|
||||
".txt": _parse_text,
|
||||
".md": _parse_text,
|
||||
|
||||
Reference in New Issue
Block a user