"""Chunker 切分器的单元测试""" from app.core.chunker import Chunker class TestStructuredChunking: """结构化文档按标题树切分""" def test_split_by_sections_and_section_path(self): """按 section 切分,section_path 为祖先标题链""" text = ( "# 安装指南\n这是简介内容,介绍安装流程。\n\n" "## 环境准备\n准备 Python 环境与依赖。\n\n" "## 安装步骤\n执行安装命令完成部署。" ) chunks = Chunker(max_chars=30).chunk(text, "doc-1") assert [c.chunk_index for c in chunks] == [0, 1, 2] assert [c.section_path for c in chunks] == ["安装指南", "安装指南 / 环境准备", "安装指南 / 安装步骤"] # chunk 文本含所属标题行本身 assert chunks[0].text.startswith("# 安装指南") assert chunks[1].text.startswith("## 环境准备") assert chunks[2].text.startswith("## 安装步骤") assert all(c.doc_id == "doc-1" for c in chunks) def test_preamble_before_first_heading(self): """首个标题前的引导正文归入空 section_path 的 chunk""" text = "无标题的引导内容,描述文档主题。\n# 第一章\n章节正文内容。" chunks = Chunker(max_chars=20).chunk(text, "doc-1") assert len(chunks) == 2 assert chunks[0].section_path == "" assert chunks[0].text == "无标题的引导内容,描述文档主题。" assert chunks[1].section_path == "第一章" def test_sibling_sections_reset_path(self): """同级标题弹栈,section_path 不含上一分支""" text = "# 甲\n内容甲。\n## 甲一\n内容甲一。\n# 乙\n内容乙。" chunks = Chunker(max_chars=20).chunk(text, "doc-1") assert [c.section_path for c in chunks] == ["甲", "甲 / 甲一", "乙"] class TestLongSectionSplitting: """超长 section 二次切分""" def test_long_section_split_by_paragraphs(self): """超长 section 按空行段落累加切分,同 section 的 chunk 共享 section_path""" para1 = "第一段内容," * 6 # 36 字符 para2 = "第二段内容," * 6 text = f"# 大章节\n{para1}\n\n{para2}" chunks = Chunker(max_chars=60).chunk(text, "doc-1") assert len(chunks) == 2 assert [c.chunk_index for c in chunks] == [0, 1] assert all(c.section_path == "大章节" for c in chunks) assert all(len(c.text) <= 60 for c in chunks) assert chunks[0].text.startswith("# 大章节") assert para1 in chunks[0].text assert para2 in chunks[1].text def test_long_paragraph_hard_split(self): """单段落仍超长时按 max_chars 硬切""" long_para = "长" * 150 text = f"# 章节\n{long_para}" chunks = Chunker(max_chars=50).chunk(text, "doc-1") # section 文本为 "# 章节\n" + 150 字 = 156 字符,硬切为 4 段 assert len(chunks) == 4 assert [c.chunk_index for c in chunks] == [0, 1, 2, 3] assert all(len(c.text) <= 50 for c in chunks) assert all(c.section_path == "章节" for c in chunks) assert chunks[0].text.startswith("# 章节") class TestPlainTextChunking: """无结构文本按段落切分""" def test_split_by_paragraphs(self): """按空行段落累加切分,section_path 为空""" para = "段落内容," * 5 # 25 字符 text = f"{para}\n\n{para}\n\n{para}" chunks = Chunker(max_chars=55).chunk(text, "doc-1") # 两段累加 52 字符 ≤ 55,再加一段超限,故切为 2 块 assert len(chunks) == 2 assert [c.chunk_index for c in chunks] == [0, 1] assert all(c.section_path == "" for c in chunks) assert all(len(c.text) <= 55 for c in chunks) def test_long_plain_text_hard_split(self): """无结构单段落超长时硬切""" text = "字" * 120 chunks = Chunker(max_chars=50).chunk(text, "doc-1") assert len(chunks) == 3 assert all(len(c.text) <= 50 for c in chunks) assert all(c.section_path == "" for c in chunks) class TestShortAndEmptyText: """短文本与空文本""" def test_short_text_single_chunk(self): """全文不超过 max_chars 时整篇单 chunk""" chunks = Chunker(max_chars=100).chunk("# 标题\n短文本内容", "doc-1") assert len(chunks) == 1 assert chunks[0].chunk_index == 0 assert chunks[0].text == "# 标题\n短文本内容" assert chunks[0].section_path == "" def test_empty_text_returns_empty(self): assert Chunker(max_chars=100).chunk("", "doc-1") == [] assert Chunker(max_chars=100).chunk(" \n ", "doc-1") == [] class TestChunkIndexContinuity: """chunk_index 跨 section 连续递增""" def test_indices_continuous_across_sections(self): para = "内容段落," * 6 # 30 字符 text = f"# 第一章\n{para}\n\n{para}\n# 第二章\n{para}\n\n{para}" chunks = Chunker(max_chars=50).chunk(text, "doc-1") # 每个 section(标题行 + 两段)超 50,各切为 2 块,共 4 块 assert len(chunks) == 4 assert [c.chunk_index for c in chunks] == [0, 1, 2, 3] assert chunks[0].section_path == chunks[1].section_path == "第一章" assert chunks[2].section_path == chunks[3].section_path == "第二章"