"""多格式文件文本提取:按扩展名分发到对应解析器(插件化) 支持的扩展名: - .txt / .md:UTF-8 解码(errors="replace" 兜底) - .html / .htm:标准库 html.parser 剥离标签提取可见文本 - .pdf:PDF 文本层提取插件(默认 pypdf);文本层为空(扫描件)时降级到 OCR 插件 - .docx:DOCX 解析插件(默认 python-docx) 插件化设计: - PdfTextExtractor / DocxParser / OcrEngine 三个 Protocol - 每种插件类型有注册表 + 默认实现 + 工厂 - 工厂根据 runtime_settings.parsers 选择具体插件 - 未注册或依赖缺失时降级到默认插件并告警 未识别扩展名抛 ValueError("不支持的文件类型: {ext}"); 解析异常统一包装为 ValueError("文件解析失败: {detail}")。 """ from __future__ import annotations import io from collections.abc import Callable from html.parser import HTMLParser from pathlib import Path from typing import Any, Protocol, runtime_checkable import structlog from app.config import settings from app.core.runtime_settings import get_runtime_settings logger = structlog.get_logger() # ---------------------------------------------------------------------------- # # HTML / Text 解析(无插件化需求,保留原实现) # ---------------------------------------------------------------------------- # class _VisibleTextExtractor(HTMLParser): """HTMLParser 子类:累积可见文本,跳过 script/style 内容""" def __init__(self) -> None: super().__init__(convert_charrefs=True) self._parts: list[str] = [] self._skip_depth = 0 # 在 script/style 标签内时 > 0 def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag.lower() in {"script", "style"}: self._skip_depth += 1 def handle_endtag(self, tag: str) -> None: if tag.lower() in {"script", "style"} and self._skip_depth > 0: self._skip_depth -= 1 def handle_data(self, data: str) -> None: if self._skip_depth == 0: self._parts.append(data) def get_text(self) -> str: text = " ".join(self._parts) return " ".join(text.split()) def _decode_html(content: bytes) -> str: try: return content.decode("utf-8-sig") except UnicodeDecodeError: return content.decode("latin-1", errors="replace") def _parse_text(content: bytes) -> str: return content.decode("utf-8", errors="replace") def _parse_html(content: bytes) -> str: parser = _VisibleTextExtractor() parser.feed(_decode_html(content)) parser.close() return parser.get_text() # ---------------------------------------------------------------------------- # # 插件协议 # ---------------------------------------------------------------------------- # @runtime_checkable class PdfTextExtractor(Protocol): """PDF 文本层提取插件协议""" def extract_text(self, content: bytes) -> str: """从 PDF 二进制内容提取文本层;无文本层返回空字符串""" ... @runtime_checkable class DocxParser(Protocol): """DOCX 解析插件协议""" def extract_text(self, content: bytes) -> str: """从 DOCX 二进制内容提取段落文本""" ... @runtime_checkable class OcrEngine(Protocol): """OCR 引擎插件协议""" def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str: """对 PDF 跑 OCR,返回识别文本;失败返回空字符串""" ... # ---------------------------------------------------------------------------- # # 默认 PDF 插件:pypdf # ---------------------------------------------------------------------------- # class PypdfTextExtractor: """pypdf 文本层提取(默认)""" def extract_text(self, content: bytes) -> str: from pypdf import PdfReader reader = PdfReader(io.BytesIO(content)) parts: list[str] = [] for page in reader.pages: text = page.extract_text() or "" if text: parts.append(text) return "\n".join(parts).strip() # ---------------------------------------------------------------------------- # # 备选 PDF 插件:pdfplumber(lazy import,依赖缺失时不可用) # ---------------------------------------------------------------------------- # class PdfplumberTextExtractor: """pdfplumber 文本层提取(备选,对复杂排版更友好)""" def extract_text(self, content: bytes) -> str: import pdfplumber # type: ignore[import-untyped] parts: list[str] = [] with pdfplumber.open(io.BytesIO(content)) as pdf: for page in pdf.pages: text = page.extract_text() or "" if text: parts.append(text) return "\n".join(parts).strip() # ---------------------------------------------------------------------------- # # 默认 DOCX 插件:python-docx # ---------------------------------------------------------------------------- # class PythonDocxParser: """python-docx 段落提取(默认)""" def extract_text(self, content: bytes) -> str: from docx import Document # type: ignore[import-untyped] document = Document(io.BytesIO(content)) parts = [p.text for p in document.paragraphs if p.text and p.text.strip()] return "\n".join(parts).strip() # ---------------------------------------------------------------------------- # # 默认 OCR 插件:rapidocr-onnxruntime + pypdfium2 # ---------------------------------------------------------------------------- # class RapidocrOcrEngine: """rapidocr-onnxruntime OCR 引擎(默认) 流程:pypdfium2 渲染每页为 PIL Image → rapidocr 识别 → 拼接。 类级懒加载单例(_engine/_unavailable 为类属性,跨实例共享), 依赖缺失时降级返回空文本。 """ _engine: Any | None = None _unavailable: bool = False def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str: if self._unavailable: return "" # 1. 懒加载 OCR 引擎(类级单例,跨实例共享) if self._engine is None: try: from rapidocr_onnxruntime import RapidOCR RapidocrOcrEngine._engine = RapidOCR() logger.info("PDF OCR 引擎已初始化", dpi=dpi) except Exception: RapidocrOcrEngine._unavailable = True logger.warning( "OCR 依赖不可用,扫描件 PDF 将无法提取文本", exc_info=True ) return "" # 2. 渲染并识别 try: import pypdfium2 as pdfium scale = max(1.0, dpi / 72.0) pdf = pdfium.PdfDocument(io.BytesIO(content)) total = min(len(pdf), max(1, max_pages)) page_texts: list[str] = [] for i in range(total): page = pdf[i] pil_image = page.render(scale=scale).to_pil() result, _ = self._engine(pil_image) if result: lines = [ item[1] for item in result if item and len(item) >= 2 and item[1] ] if lines: page_texts.append("\n".join(lines)) pdf.close() return "\n".join(page_texts).strip() except Exception as exc: logger.warning( "PDF OCR 失败,降级返回空文本", error=str(exc), exc_info=True ) return "" # ---------------------------------------------------------------------------- # # 备选 OCR 插件:tesseract(lazy import,依赖缺失时不可用) # ---------------------------------------------------------------------------- # class TesseractOcrEngine: """pytesseract + pdfium OCR(备选) 需要系统安装 tesseract 二进制与语言包。 类级 _unavailable 单例,跨实例共享。 """ _unavailable: bool = False def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str: if self._unavailable: return "" try: import pypdfium2 as pdfium import pytesseract # type: ignore[import-untyped] from PIL import Image # type: ignore[import-untyped] except Exception: TesseractOcrEngine._unavailable = True logger.warning("tesseract 依赖不可用,降级返回空文本", exc_info=True) return "" try: scale = max(1.0, dpi / 72.0) pdf = pdfium.PdfDocument(io.BytesIO(content)) total = min(len(pdf), max(1, max_pages)) page_texts: list[str] = [] for i in range(total): page = pdf[i] pil_image = page.render(scale=scale).to_pil() text = pytesseract.image_to_string(pil_image, lang="chi_sim+eng") if text: page_texts.append(text) pdf.close() return "\n".join(page_texts).strip() except Exception as exc: logger.warning( "tesseract OCR 失败,降级返回空文本", error=str(exc), exc_info=True ) return "" class NoopOcrEngine: """关闭 OCR 的占位插件(strategy=none 时使用)""" def ocr_pdf(self, content: bytes, *, max_pages: int, dpi: int) -> str: return "" # ---------------------------------------------------------------------------- # # 插件注册表 # ---------------------------------------------------------------------------- # _PDF_PLUGINS: dict[str, type[PdfTextExtractor]] = { "pypdf": PypdfTextExtractor, "pdfplumber": PdfplumberTextExtractor, } _DOCX_PLUGINS: dict[str, type[DocxParser]] = { "python_docx": PythonDocxParser, } _OCR_PLUGINS: dict[str, type[OcrEngine]] = { "rapidocr": RapidocrOcrEngine, "tesseract": TesseractOcrEngine, "none": NoopOcrEngine, } def list_pdf_plugins() -> list[str]: return list(_PDF_PLUGINS.keys()) def list_docx_plugins() -> list[str]: return list(_DOCX_PLUGINS.keys()) def list_ocr_plugins() -> list[str]: return list(_OCR_PLUGINS.keys()) # ---------------------------------------------------------------------------- # # 工厂 # ---------------------------------------------------------------------------- # # 进程级插件单例缓存 _pdf_plugin_cache: dict[str, PdfTextExtractor] = {} _docx_plugin_cache: dict[str, DocxParser] = {} _ocr_plugin_cache: dict[str, OcrEngine] = {} def _get_pdf_plugin() -> PdfTextExtractor: rt = get_runtime_settings() name = rt.parsers.pdf.plugin or "pypdf" if name in _pdf_plugin_cache: return _pdf_plugin_cache[name] cls = _PDF_PLUGINS.get(name) if cls is None: logger.warning("未知 PDF 插件,降级到 pypdf", requested=name) cls = PypdfTextExtractor name = "pypdf" instance = cls() _pdf_plugin_cache[name] = instance return instance def _get_docx_plugin() -> DocxParser: rt = get_runtime_settings() name = rt.parsers.docx.plugin or "python_docx" if name in _docx_plugin_cache: return _docx_plugin_cache[name] cls = _DOCX_PLUGINS.get(name) if cls is None: logger.warning("未知 DOCX 插件,降级到 python_docx", requested=name) cls = PythonDocxParser name = "python_docx" instance = cls() _docx_plugin_cache[name] = instance return instance def _get_ocr_plugin() -> OcrEngine: rt = get_runtime_settings() name = rt.parsers.ocr.plugin or "rapidocr" if name in _ocr_plugin_cache: return _ocr_plugin_cache[name] cls = _OCR_PLUGINS.get(name) if cls is None: logger.warning("未知 OCR 插件,降级到 rapidocr", requested=name) cls = RapidocrOcrEngine name = "rapidocr" instance = cls() _ocr_plugin_cache[name] = instance return instance def invalidate_parser_plugin_cache(kind: str | None = None) -> None: """清除插件缓存(runtime_settings 更新后调用)""" if kind is None or kind == "pdf": _pdf_plugin_cache.clear() if kind is None or kind == "docx": _docx_plugin_cache.clear() if kind is None or kind == "ocr": _ocr_plugin_cache.clear() # 重置 RapidocrOcrEngine 类级标记,允许重新初始化 RapidocrOcrEngine._engine = None RapidocrOcrEngine._unavailable = False TesseractOcrEngine._unavailable = False # ---------------------------------------------------------------------------- # # 顶层解析函数 # ---------------------------------------------------------------------------- # def _parse_pdf(content: bytes) -> str: """PDF 解析:优先文本层插件;文本层为空时降级 OCR 插件""" rt = get_runtime_settings() pdf_plugin = _get_pdf_plugin() try: text_layer = pdf_plugin.extract_text(content) except Exception as exc: logger.warning( "PDF 文本层提取失败,尝试 OCR 降级", plugin=rt.parsers.pdf.plugin, error=str(exc), exc_info=True, ) text_layer = "" if text_layer: return text_layer # 文本层为空 → OCR 降级 if not settings.pdf_ocr_enabled: return "" ocr_plugin = _get_ocr_plugin() return ocr_plugin.ocr_pdf( content, max_pages=rt.parsers.ocr.params.get("max_pages", settings.pdf_ocr_max_pages), dpi=rt.parsers.ocr.params.get("dpi", settings.pdf_ocr_dpi), ) def _parse_docx(content: bytes) -> str: """DOCX 解析""" return _get_docx_plugin().extract_text(content) # 扩展名 → 解析函数映射 _PARSERS: dict[str, Callable[[bytes], str]] = { ".txt": _parse_text, ".md": _parse_text, ".html": _parse_html, ".htm": _parse_html, ".pdf": _parse_pdf, ".docx": _parse_docx, } def parse_file(filename: str, content: bytes) -> str: """按扩展名分发解析器提取文本 Args: filename: 文件名(用于判定扩展名) content: 文件二进制内容 Returns: 提取的纯文本 Raises: ValueError: 未识别扩展名 → "不支持的文件类型: {ext}" 解析失败 → "文件解析失败: {detail}"(保留原异常链) """ ext = Path(filename).suffix.lower() parser = _PARSERS.get(ext) if parser is None: raise ValueError(f"不支持的文件类型: {ext or '(无扩展名)'}") try: return parser(content) except ValueError: raise except Exception as exc: raise ValueError(f"文件解析失败: {exc}") from exc def supported_extensions() -> set[str]: """返回当前支持的扩展名集合(含点号,全小写)""" return set(_PARSERS.keys())