kplam
51dc8dc4f6
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
131 lines
5.3 KiB
Python
131 lines
5.3 KiB
Python
"""Chunker 切分器的单元测试"""
|
|
|
|
from app.core.chunker import Chunker
|
|
|
|
|
|
class TestStructuredChunking:
|
|
"""结构化文档按标题树切分"""
|
|
|
|
def test_split_by_sections_and_section_path(self):
|
|
"""按 section 切分，section_path 为祖先标题链"""
|
|
text = (
|
|
"# 安装指南\n这是简介内容，介绍安装流程。\n\n"
|
|
"## 环境准备\n准备 Python 环境与依赖。\n\n"
|
|
"## 安装步骤\n执行安装命令完成部署。"
|
|
)
|
|
chunks = Chunker(max_chars=30).chunk(text, "doc-1")
|
|
|
|
assert [c.chunk_index for c in chunks] == [0, 1, 2]
|
|
assert [c.section_path for c in chunks] == ["安装指南", "安装指南 / 环境准备", "安装指南 / 安装步骤"]
|
|
# chunk 文本含所属标题行本身
|
|
assert chunks[0].text.startswith("# 安装指南")
|
|
assert chunks[1].text.startswith("## 环境准备")
|
|
assert chunks[2].text.startswith("## 安装步骤")
|
|
assert all(c.doc_id == "doc-1" for c in chunks)
|
|
|
|
def test_preamble_before_first_heading(self):
|
|
"""首个标题前的引导正文归入空 section_path 的 chunk"""
|
|
text = "无标题的引导内容，描述文档主题。\n# 第一章\n章节正文内容。"
|
|
chunks = Chunker(max_chars=20).chunk(text, "doc-1")
|
|
|
|
assert len(chunks) == 2
|
|
assert chunks[0].section_path == ""
|
|
assert chunks[0].text == "无标题的引导内容，描述文档主题。"
|
|
assert chunks[1].section_path == "第一章"
|
|
|
|
def test_sibling_sections_reset_path(self):
|
|
"""同级标题弹栈，section_path 不含上一分支"""
|
|
text = "# 甲\n内容甲。\n## 甲一\n内容甲一。\n# 乙\n内容乙。"
|
|
chunks = Chunker(max_chars=20).chunk(text, "doc-1")
|
|
|
|
assert [c.section_path for c in chunks] == ["甲", "甲 / 甲一", "乙"]
|
|
|
|
|
|
class TestLongSectionSplitting:
|
|
"""超长 section 二次切分"""
|
|
|
|
def test_long_section_split_by_paragraphs(self):
|
|
"""超长 section 按空行段落累加切分，同 section 的 chunk 共享 section_path"""
|
|
para1 = "第一段内容，" * 6 # 36 字符
|
|
para2 = "第二段内容，" * 6
|
|
text = f"# 大章节\n{para1}\n\n{para2}"
|
|
chunks = Chunker(max_chars=60).chunk(text, "doc-1")
|
|
|
|
assert len(chunks) == 2
|
|
assert [c.chunk_index for c in chunks] == [0, 1]
|
|
assert all(c.section_path == "大章节" for c in chunks)
|
|
assert all(len(c.text) <= 60 for c in chunks)
|
|
assert chunks[0].text.startswith("# 大章节")
|
|
assert para1 in chunks[0].text
|
|
assert para2 in chunks[1].text
|
|
|
|
def test_long_paragraph_hard_split(self):
|
|
"""单段落仍超长时按 max_chars 硬切"""
|
|
long_para = "长" * 150
|
|
text = f"# 章节\n{long_para}"
|
|
chunks = Chunker(max_chars=50).chunk(text, "doc-1")
|
|
|
|
# section 文本为 "# 章节\n" + 150 字 = 156 字符，硬切为 4 段
|
|
assert len(chunks) == 4
|
|
assert [c.chunk_index for c in chunks] == [0, 1, 2, 3]
|
|
assert all(len(c.text) <= 50 for c in chunks)
|
|
assert all(c.section_path == "章节" for c in chunks)
|
|
assert chunks[0].text.startswith("# 章节")
|
|
|
|
|
|
class TestPlainTextChunking:
|
|
"""无结构文本按段落切分"""
|
|
|
|
def test_split_by_paragraphs(self):
|
|
"""按空行段落累加切分，section_path 为空"""
|
|
para = "段落内容，" * 5 # 25 字符
|
|
text = f"{para}\n\n{para}\n\n{para}"
|
|
chunks = Chunker(max_chars=55).chunk(text, "doc-1")
|
|
|
|
# 两段累加 52 字符 ≤ 55，再加一段超限，故切为 2 块
|
|
assert len(chunks) == 2
|
|
assert [c.chunk_index for c in chunks] == [0, 1]
|
|
assert all(c.section_path == "" for c in chunks)
|
|
assert all(len(c.text) <= 55 for c in chunks)
|
|
|
|
def test_long_plain_text_hard_split(self):
|
|
"""无结构单段落超长时硬切"""
|
|
text = "字" * 120
|
|
chunks = Chunker(max_chars=50).chunk(text, "doc-1")
|
|
|
|
assert len(chunks) == 3
|
|
assert all(len(c.text) <= 50 for c in chunks)
|
|
assert all(c.section_path == "" for c in chunks)
|
|
|
|
|
|
class TestShortAndEmptyText:
|
|
"""短文本与空文本"""
|
|
|
|
def test_short_text_single_chunk(self):
|
|
"""全文不超过 max_chars 时整篇单 chunk"""
|
|
chunks = Chunker(max_chars=100).chunk("# 标题\n短文本内容", "doc-1")
|
|
|
|
assert len(chunks) == 1
|
|
assert chunks[0].chunk_index == 0
|
|
assert chunks[0].text == "# 标题\n短文本内容"
|
|
assert chunks[0].section_path == ""
|
|
|
|
def test_empty_text_returns_empty(self):
|
|
assert Chunker(max_chars=100).chunk("", "doc-1") == []
|
|
assert Chunker(max_chars=100).chunk(" \n ", "doc-1") == []
|
|
|
|
|
|
class TestChunkIndexContinuity:
|
|
"""chunk_index 跨 section 连续递增"""
|
|
|
|
def test_indices_continuous_across_sections(self):
|
|
para = "内容段落，" * 6 # 30 字符
|
|
text = f"# 第一章\n{para}\n\n{para}\n# 第二章\n{para}\n\n{para}"
|
|
chunks = Chunker(max_chars=50).chunk(text, "doc-1")
|
|
|
|
# 每个 section（标题行 + 两段）超 50，各切为 2 块，共 4 块
|
|
assert len(chunks) == 4
|
|
assert [c.chunk_index for c in chunks] == [0, 1, 2, 3]
|
|
assert chunks[0].section_path == chunks[1].section_path == "第一章"
|
|
assert chunks[2].section_path == chunks[3].section_path == "第二章"
|