World
" result = parse_file("page.html", html) assert "Hello" in result assert "World" in result assert "<" not in result assert ">" not in result assert "x=1" not in result assert "p{}" not in result def test_parse_htm_same_as_html() -> None: """htm：与 html 走同一解析器""" html = b"same content
" assert parse_file("page.htm", html) == parse_file("page.html", html) def test_parse_pdf_extracts_text() -> None: """pdf：从最小 PDF 中提取文本""" pdf_bytes = _make_minimal_pdf("Hello PDF World") result = parse_file("doc.pdf", pdf_bytes) assert "Hello PDF World" in result def test_parse_docx_extracts_paragraphs() -> None: """docx：提取段落文本""" docx_bytes = _make_docx(["第一段落", "第二段落"]) result = parse_file("doc.docx", docx_bytes) assert "第一段落" in result assert "第二段落" in result def test_parse_file_unsupported_extension_raises() -> None: """不支持的扩展名抛 ValueError，message 含扩展名""" with pytest.raises(ValueError, match=r"不支持的文件类型: \.xlsx"): parse_file("data.xlsx", b"binary") def test_parse_file_no_extension_raises() -> None: """无扩展名抛 ValueError""" with pytest.raises(ValueError, match=r"不支持的文件类型"): parse_file("noext", b"text") def test_parse_file_corrupted_pdf_raises() -> None: """损坏的 PDF 抛 ValueError，message 含'文件解析失败'""" with pytest.raises(ValueError, match=r"文件解析失败"): parse_file("bad.pdf", b"not a real pdf") def test_parse_file_empty_html_returns_empty_string() -> None: """空 HTML 返回空字符串""" assert parse_file("empty.html", b"") == "" def test_parse_file_html_with_script_style_excluded() -> None: """HTML 中 script/style 内容被排除""" html = b"visible
" result = parse_file("page.html", html) assert "visible" in result assert "alert" not in result assert "body{}" not in result def test_parse_file_html_with_entities_decoded() -> None: """HTML 实体被解码""" html = b"Tom & Jerry
" result = parse_file("page.html", html) assert "Tom & Jerry" in result assert "&" not in result def test_supported_extensions_contains_expected_set() -> None: """supported_extensions 返回包含全部六种扩展名的集合""" exts = supported_extensions() assert {".txt", ".md", ".html", ".htm", ".pdf", ".docx"} <= exts # --------------------------------------------------------------------------- # # PDF OCR 降级路径测试（mock pypdf / pypdfium2 / rapidocr_onnxruntime，不真实下载模型） # --------------------------------------------------------------------------- # @pytest.fixture(autouse=True) def _reset_ocr_state() -> Any: """每个 OCR 测试前后重置模块级 OCR 引擎状态，避免相互污染""" saved_engine = fp_module._ocr_engine saved_unavailable = fp_module._ocr_unavailable yield fp_module._ocr_engine = saved_engine fp_module._ocr_unavailable = saved_unavailable class _FakeTextPage: """pypdf PageObject 替身：返回固定文本""" def __init__(self, text: str) -> None: self._text = text def extract_text(self) -> str: return self._text class _FakePdfReader: """pypdf.PdfReader 替身：构造时不解析，按预设页文本返回""" def __init__(self, stream: Any) -> None: self.pages = [_FakeTextPage(""), _FakeTextPage("")] class _FakePilImage: pass class _FakeRenderResult: def to_pil(self) -> _FakePilImage: return _FakePilImage() class _FakePdfiumPage: def render(self, scale: float) -> _FakeRenderResult: return _FakeRenderResult() class _FakePdfDocument: """pypdfium2.PdfDocument 替身""" def __init__(self, stream: Any) -> None: self._n_pages = 2 def __len__(self) -> int: return self._n_pages def __getitem__(self, i: int) -> _FakePdfiumPage: return _FakePdfiumPage() def close(self) -> None: pass class _FakeOcrEngine: """rapidocr RapidOCR 替身：每次返回固定识别结果""" def __init__(self) -> None: self.call_count = 0 def __call__(self, image: Any) -> tuple[list[list[Any]], float]: self.call_count += 1 # 返回 [[box, text, score], ...] 结构 return [[[0, 0], f"OCR文本第{self.call_count}页", 0.95]], 0.1 def _patch_pdf_ocr_deps(monkeypatch: pytest.MonkeyPatch) -> None: """把 _parse_pdf/_ocr_pdf 内部用到的 pypdf / pypdfium2 / rapidocr_onnxruntime 全部替换""" monkeypatch.setattr("pypdf.PdfReader", _FakePdfReader) monkeypatch.setattr("pypdfium2.PdfDocument", _FakePdfDocument) fake_module = type(sys)("rapidocr_onnxruntime") fake_module.RapidOCR = _FakeOcrEngine monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", fake_module) def test_parse_pdf_ocr_fallback_when_text_layer_empty(monkeypatch: pytest.MonkeyPatch) -> None: """扫描件 PDF（文本层全空）触发 OCR 降级，返回识别文本""" _patch_pdf_ocr_deps(monkeypatch) monkeypatch.setattr(settings, "pdf_ocr_enabled", True) monkeypatch.setattr(settings, "pdf_ocr_max_pages", 30) monkeypatch.setattr(settings, "pdf_ocr_dpi", 200) result = parse_file("scan.pdf", b"fake pdf bytes") # 两页都跑了 OCR，每页返回一段文本 assert "OCR文本第1页" in result assert "OCR文本第2页" in result def test_parse_pdf_ocr_skipped_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: """pdf_ocr_enabled=False：文本层为空时直接返回空，不调 OCR""" _patch_pdf_ocr_deps(monkeypatch) monkeypatch.setattr(settings, "pdf_ocr_enabled", False) result = parse_file("scan.pdf", b"fake pdf bytes") assert result == "" def test_parse_pdf_ocr_respects_max_pages(monkeypatch: pytest.MonkeyPatch) -> None: """max_pages=1：只 OCR 第一页，第二页跳过""" _patch_pdf_ocr_deps(monkeypatch) monkeypatch.setattr(settings, "pdf_ocr_enabled", True) monkeypatch.setattr(settings, "pdf_ocr_max_pages", 1) monkeypatch.setattr(settings, "pdf_ocr_dpi", 200) result = parse_file("scan.pdf", b"fake pdf bytes") assert "OCR文本第1页" in result assert "OCR文本第2页" not in result def test_parse_pdf_ocr_returns_empty_when_dependency_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: """rapidocr 导入失败：降级返回空文本，且把 _ocr_unavailable 置 True 避免重试""" monkeypatch.setattr("pypdf.PdfReader", _FakePdfReader) # 故意让 rapidocr_onnxruntime 提供一个非类的 RapidOCR，构造时抛错 fake_module = type(sys)("rapidocr_onnxruntime") def _boom(*args: Any, **kwargs: Any) -> None: raise RuntimeError("model missing") fake_module.RapidOCR = _boom # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", fake_module) monkeypatch.setattr(settings, "pdf_ocr_enabled", True) assert parse_file("scan.pdf", b"fake pdf bytes") == "" assert fp_module._ocr_unavailable is True def test_parse_pdf_ocr_runtime_exception_falls_back_to_empty(monkeypatch: pytest.MonkeyPatch) -> None: """OCR 运行时抛错：仅告警，降级返回空文本（不抛出 ValueError）""" monkeypatch.setattr("pypdf.PdfReader", _FakePdfReader) class _ExplodingPdfDocument: def __init__(self, stream: Any) -> None: raise RuntimeError("pdfium render failed") monkeypatch.setattr("pypdfium2.PdfDocument", _ExplodingPdfDocument) fake_module = type(sys)("rapidocr_onnxruntime") fake_module.RapidOCR = _FakeOcrEngine monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", fake_module) monkeypatch.setattr(settings, "pdf_ocr_enabled", True) assert parse_file("scan.pdf", b"fake pdf bytes") == "" def test_parse_pdf_text_layer_present_skips_ocr(monkeypatch: pytest.MonkeyPatch) -> None: """文本层非空：直接返回文本，OCR 引擎不会被实例化""" call_count = 0 class _CountingReader: def __init__(self, stream: Any) -> None: self.pages = [_FakeTextPage("这是文本层的内容")] monkeypatch.setattr("pypdf.PdfReader", _CountingReader) # 即便 OCR 依赖故意坏掉，也不应被调用 bad_module = type(sys)("rapidocr_onnxruntime") bad_module.RapidOCR = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("不应被调用")) # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "rapidocr_onnxruntime", bad_module) monkeypatch.setattr(settings, "pdf_ocr_enabled", True) result = parse_file("text.pdf", b"fake pdf bytes") assert result == "这是文本层的内容" assert fp_module._ocr_engine is None