kplam
2ab8b56a01
此提交实现了完整的知识库管理系统： 1. 新增Vue3 + Antd Vue前端管理后台，包含登录、文档管理、检索、类目设置等完整页面 2. 重构后端LLM调用抽象层，支持Ollama与OpenAI兼容服务动态切换 3. 调整默认嵌入模型配置为本地bge-m3模式 4. 优化入库任务去重逻辑与缓存清理机制 5. 完善Docker镜像构建与docker-compose部署配置 6. 修复多项测试用例与兼容性问题 7. 新增运行时配置API，支持动态调整系统参数
468 lines
15 KiB
Python
468 lines
15 KiB
Python
"""多格式文件文本提取：按扩展名分发到对应解析器（插件化）
|
||
|
||
支持的扩展名：
|
||
- .txt / .md：UTF-8 解码（errors="replace" 兜底）
|
||
- .html / .htm：标准库 html.parser 剥离标签提取可见文本
|
||
- .pdf：PDF 文本层提取插件（默认 pypdf）；文本层为空（扫描件）时降级到 OCR 插件
|
||
- .docx：DOCX 解析插件（默认 python-docx）
|
||
|
||
插件化设计：
|
||
- PdfTextExtractor / DocxParser / OcrEngine 三个 Protocol
|
||
- 每种插件类型有注册表 + 默认实现 + 工厂
|
||
- 工厂根据 runtime_settings.parsers 选择具体插件
|
||
- 未注册或依赖缺失时降级到默认插件并告警
|
||
|
||
未识别扩展名抛 ValueError("不支持的文件类型: {ext}")；
|
||
解析异常统一包装为 ValueError("文件解析失败: {detail}")。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
from collections.abc import Callable
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
from typing import Any, Protocol, runtime_checkable
|
||
|
||
import structlog
|
||
|
||
from app.config import settings
|
||
from app.core.runtime_settings import get_runtime_settings
|
||
|
||
logger = structlog.get_logger()
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# HTML / Text 解析（无插件化需求，保留原实现）
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
class _VisibleTextExtractor(HTMLParser):
|
||
"""HTMLParser 子类：累积可见文本，跳过 script/style 内容"""
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self._parts: list[str] = []
|
||
self._skip_depth = 0 # 在 script/style 标签内时 > 0
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
if tag.lower() in {"script", "style"}:
|
||
self._skip_depth += 1
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
if tag.lower() in {"script", "style"} and self._skip_depth > 0:
|
||
self._skip_depth -= 1
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
if self._skip_depth == 0:
|
||
self._parts.append(data)
|
||
|
||
def get_text(self) -> str:
|
||
text = " ".join(self._parts)
|
||
return " ".join(text.split())
|
||
|
||
|
||
def _decode_html(content: bytes) -> str:
|
||
try:
|
||
return content.decode("utf-8-sig")
|
||
except UnicodeDecodeError:
|
||
return content.decode("latin-1", errors="replace")
|
||
|
||
|
||
def _parse_text(content: bytes) -> str:
|
||
return content.decode("utf-8", errors="replace")
|
||
|
||
|
||
def _parse_html(content: bytes) -> str:
|
||
parser = _VisibleTextExtractor()
|
||
parser.feed(_decode_html(content))
|
||
parser.close()
|
||
return parser.get_text()
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 插件协议
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
@runtime_checkable
|
||
class PdfTextExtractor(Protocol):
|
||
"""PDF 文本层提取插件协议"""
|
||
|
||
def extract_text(self, content: bytes) -> str:
|
||
"""从 PDF 二进制内容提取文本层；无文本层返回空字符串"""
|
||
...
|
||
|
||
|
||
@runtime_checkable
|
||
class DocxParser(Protocol):
|
||
"""DOCX 解析插件协议"""
|
||
|
||
def extract_text(self, content: bytes) -> str:
|
||
"""从 DOCX 二进制内容提取段落文本"""
|
||
...
|
||
|
||
|
||
@runtime_checkable
|
||
class OcrEngine(Protocol):
|
||
"""OCR 引擎插件协议"""
|
||
|
||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||
"""对 PDF 跑 OCR，返回识别文本；失败返回空字符串"""
|
||
...
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 默认 PDF 插件：pypdf
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
class PypdfTextExtractor:
|
||
"""pypdf 文本层提取（默认）"""
|
||
|
||
def extract_text(self, content: bytes) -> str:
|
||
from pypdf import PdfReader
|
||
|
||
reader = PdfReader(io.BytesIO(content))
|
||
parts: list[str] = []
|
||
for page in reader.pages:
|
||
text = page.extract_text() or ""
|
||
if text:
|
||
parts.append(text)
|
||
return "\n".join(parts).strip()
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 备选 PDF 插件：pdfplumber（lazy import，依赖缺失时不可用）
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
class PdfplumberTextExtractor:
|
||
"""pdfplumber 文本层提取（备选，对复杂排版更友好）"""
|
||
|
||
def extract_text(self, content: bytes) -> str:
|
||
import pdfplumber # type: ignore[import-untyped]
|
||
|
||
parts: list[str] = []
|
||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||
for page in pdf.pages:
|
||
text = page.extract_text() or ""
|
||
if text:
|
||
parts.append(text)
|
||
return "\n".join(parts).strip()
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 默认 DOCX 插件：python-docx
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
class PythonDocxParser:
|
||
"""python-docx 段落提取（默认）"""
|
||
|
||
def extract_text(self, content: bytes) -> str:
|
||
from docx import Document # type: ignore[import-untyped]
|
||
|
||
document = Document(io.BytesIO(content))
|
||
parts = [p.text for p in document.paragraphs if p.text and p.text.strip()]
|
||
return "\n".join(parts).strip()
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 默认 OCR 插件：rapidocr-onnxruntime + pypdfium2
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
class RapidocrOcrEngine:
|
||
"""rapidocr-onnxruntime OCR 引擎（默认）
|
||
|
||
流程：pypdfium2 渲染每页为 PIL Image → rapidocr 识别 → 拼接。
|
||
类级懒加载单例（_engine/_unavailable 为类属性，跨实例共享），
|
||
依赖缺失时降级返回空文本。
|
||
"""
|
||
|
||
_engine: Any | None = None
|
||
_unavailable: bool = False
|
||
|
||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||
if self._unavailable:
|
||
return ""
|
||
|
||
# 1. 懒加载 OCR 引擎（类级单例，跨实例共享）
|
||
if self._engine is None:
|
||
try:
|
||
from rapidocr_onnxruntime import RapidOCR
|
||
|
||
RapidocrOcrEngine._engine = RapidOCR()
|
||
logger.info("PDF OCR 引擎已初始化", dpi=dpi)
|
||
except Exception:
|
||
RapidocrOcrEngine._unavailable = True
|
||
logger.warning(
|
||
"OCR 依赖不可用，扫描件 PDF 将无法提取文本", exc_info=True
|
||
)
|
||
return ""
|
||
|
||
# 2. 渲染并识别
|
||
try:
|
||
import pypdfium2 as pdfium
|
||
|
||
scale = max(1.0, dpi / 72.0)
|
||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||
total = min(len(pdf), max(1, max_pages))
|
||
page_texts: list[str] = []
|
||
for i in range(total):
|
||
page = pdf[i]
|
||
pil_image = page.render(scale=scale).to_pil()
|
||
result, _ = self._engine(pil_image)
|
||
if result:
|
||
lines = [
|
||
item[1]
|
||
for item in result
|
||
if item and len(item) >= 2 and item[1]
|
||
]
|
||
if lines:
|
||
page_texts.append("\n".join(lines))
|
||
pdf.close()
|
||
return "\n".join(page_texts).strip()
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"PDF OCR 失败，降级返回空文本", error=str(exc), exc_info=True
|
||
)
|
||
return ""
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 备选 OCR 插件：tesseract（lazy import，依赖缺失时不可用）
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
class TesseractOcrEngine:
|
||
"""pytesseract + pdfium OCR（备选）
|
||
|
||
需要系统安装 tesseract 二进制与语言包。
|
||
类级 _unavailable 单例，跨实例共享。
|
||
"""
|
||
|
||
_unavailable: bool = False
|
||
|
||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||
if self._unavailable:
|
||
return ""
|
||
try:
|
||
import pypdfium2 as pdfium
|
||
import pytesseract # type: ignore[import-untyped]
|
||
from PIL import Image # type: ignore[import-untyped]
|
||
except Exception:
|
||
TesseractOcrEngine._unavailable = True
|
||
logger.warning("tesseract 依赖不可用，降级返回空文本", exc_info=True)
|
||
return ""
|
||
|
||
try:
|
||
scale = max(1.0, dpi / 72.0)
|
||
pdf = pdfium.PdfDocument(io.BytesIO(content))
|
||
total = min(len(pdf), max(1, max_pages))
|
||
page_texts: list[str] = []
|
||
for i in range(total):
|
||
page = pdf[i]
|
||
pil_image = page.render(scale=scale).to_pil()
|
||
text = pytesseract.image_to_string(pil_image, lang="chi_sim+eng")
|
||
if text:
|
||
page_texts.append(text)
|
||
pdf.close()
|
||
return "\n".join(page_texts).strip()
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"tesseract OCR 失败，降级返回空文本", error=str(exc), exc_info=True
|
||
)
|
||
return ""
|
||
|
||
|
||
class NoopOcrEngine:
|
||
"""关闭 OCR 的占位插件（strategy=none 时使用）"""
|
||
|
||
def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str:
|
||
return ""
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 插件注册表
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
_PDF_PLUGINS: dict[str, type[PdfTextExtractor]] = {
|
||
"pypdf": PypdfTextExtractor,
|
||
"pdfplumber": PdfplumberTextExtractor,
|
||
}
|
||
|
||
_DOCX_PLUGINS: dict[str, type[DocxParser]] = {
|
||
"python_docx": PythonDocxParser,
|
||
}
|
||
|
||
_OCR_PLUGINS: dict[str, type[OcrEngine]] = {
|
||
"rapidocr": RapidocrOcrEngine,
|
||
"tesseract": TesseractOcrEngine,
|
||
"none": NoopOcrEngine,
|
||
}
|
||
|
||
|
||
def list_pdf_plugins() -> list[str]:
|
||
return list(_PDF_PLUGINS.keys())
|
||
|
||
|
||
def list_docx_plugins() -> list[str]:
|
||
return list(_DOCX_PLUGINS.keys())
|
||
|
||
|
||
def list_ocr_plugins() -> list[str]:
|
||
return list(_OCR_PLUGINS.keys())
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 工厂
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
# 进程级插件单例缓存
|
||
_pdf_plugin_cache: dict[str, PdfTextExtractor] = {}
|
||
_docx_plugin_cache: dict[str, DocxParser] = {}
|
||
_ocr_plugin_cache: dict[str, OcrEngine] = {}
|
||
|
||
|
||
def _get_pdf_plugin() -> PdfTextExtractor:
|
||
rt = get_runtime_settings()
|
||
name = rt.parsers.pdf.plugin or "pypdf"
|
||
if name in _pdf_plugin_cache:
|
||
return _pdf_plugin_cache[name]
|
||
cls = _PDF_PLUGINS.get(name)
|
||
if cls is None:
|
||
logger.warning("未知 PDF 插件，降级到 pypdf", requested=name)
|
||
cls = PypdfTextExtractor
|
||
name = "pypdf"
|
||
instance = cls()
|
||
_pdf_plugin_cache[name] = instance
|
||
return instance
|
||
|
||
|
||
def _get_docx_plugin() -> DocxParser:
|
||
rt = get_runtime_settings()
|
||
name = rt.parsers.docx.plugin or "python_docx"
|
||
if name in _docx_plugin_cache:
|
||
return _docx_plugin_cache[name]
|
||
cls = _DOCX_PLUGINS.get(name)
|
||
if cls is None:
|
||
logger.warning("未知 DOCX 插件，降级到 python_docx", requested=name)
|
||
cls = PythonDocxParser
|
||
name = "python_docx"
|
||
instance = cls()
|
||
_docx_plugin_cache[name] = instance
|
||
return instance
|
||
|
||
|
||
def _get_ocr_plugin() -> OcrEngine:
|
||
rt = get_runtime_settings()
|
||
name = rt.parsers.ocr.plugin or "rapidocr"
|
||
if name in _ocr_plugin_cache:
|
||
return _ocr_plugin_cache[name]
|
||
cls = _OCR_PLUGINS.get(name)
|
||
if cls is None:
|
||
logger.warning("未知 OCR 插件，降级到 rapidocr", requested=name)
|
||
cls = RapidocrOcrEngine
|
||
name = "rapidocr"
|
||
instance = cls()
|
||
_ocr_plugin_cache[name] = instance
|
||
return instance
|
||
|
||
|
||
def invalidate_parser_plugin_cache(kind: str | None = None) -> None:
|
||
"""清除插件缓存（runtime_settings 更新后调用）"""
|
||
if kind is None or kind == "pdf":
|
||
_pdf_plugin_cache.clear()
|
||
if kind is None or kind == "docx":
|
||
_docx_plugin_cache.clear()
|
||
if kind is None or kind == "ocr":
|
||
_ocr_plugin_cache.clear()
|
||
# 重置 RapidocrOcrEngine 类级标记，允许重新初始化
|
||
RapidocrOcrEngine._engine = None
|
||
RapidocrOcrEngine._unavailable = False
|
||
TesseractOcrEngine._unavailable = False
|
||
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# 顶层解析函数
|
||
# ---------------------------------------------------------------------------- #
|
||
|
||
|
||
def _parse_pdf(content: bytes) -> str:
|
||
"""PDF 解析：优先文本层插件；文本层为空时降级 OCR 插件"""
|
||
rt = get_runtime_settings()
|
||
pdf_plugin = _get_pdf_plugin()
|
||
try:
|
||
text_layer = pdf_plugin.extract_text(content)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"PDF 文本层提取失败，尝试 OCR 降级",
|
||
plugin=rt.parsers.pdf.plugin,
|
||
error=str(exc),
|
||
exc_info=True,
|
||
)
|
||
text_layer = ""
|
||
|
||
if text_layer:
|
||
return text_layer
|
||
|
||
# 文本层为空 → OCR 降级
|
||
if not settings.pdf_ocr_enabled:
|
||
return ""
|
||
|
||
ocr_plugin = _get_ocr_plugin()
|
||
return ocr_plugin.ocr_pdf(
|
||
content,
|
||
max_pages=rt.parsers.ocr.params.get("max_pages", settings.pdf_ocr_max_pages),
|
||
dpi=rt.parsers.ocr.params.get("dpi", settings.pdf_ocr_dpi),
|
||
)
|
||
|
||
|
||
def _parse_docx(content: bytes) -> str:
|
||
"""DOCX 解析"""
|
||
return _get_docx_plugin().extract_text(content)
|
||
|
||
|
||
# 扩展名 → 解析函数映射
|
||
_PARSERS: dict[str, Callable[[bytes], str]] = {
|
||
".txt": _parse_text,
|
||
".md": _parse_text,
|
||
".html": _parse_html,
|
||
".htm": _parse_html,
|
||
".pdf": _parse_pdf,
|
||
".docx": _parse_docx,
|
||
}
|
||
|
||
|
||
def parse_file(filename: str, content: bytes) -> str:
|
||
"""按扩展名分发解析器提取文本
|
||
|
||
Args:
|
||
filename: 文件名（用于判定扩展名）
|
||
content: 文件二进制内容
|
||
|
||
Returns:
|
||
提取的纯文本
|
||
|
||
Raises:
|
||
ValueError: 未识别扩展名 → "不支持的文件类型: {ext}"
|
||
解析失败 → "文件解析失败: {detail}"（保留原异常链）
|
||
"""
|
||
ext = Path(filename).suffix.lower()
|
||
parser = _PARSERS.get(ext)
|
||
if parser is None:
|
||
raise ValueError(f"不支持的文件类型: {ext or '(无扩展名)'}")
|
||
try:
|
||
return parser(content)
|
||
except ValueError:
|
||
raise
|
||
except Exception as exc:
|
||
raise ValueError(f"文件解析失败: {exc}") from exc
|
||
|
||
|
||
def supported_extensions() -> set[str]:
|
||
"""返回当前支持的扩展名集合（含点号，全小写）"""
|
||
return set(_PARSERS.keys())
|