kplam
51dc8dc4f6
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
58 lines
2.3 KiB
Python
58 lines
2.3 KiB
Python
"""稀疏向量编码器单元测试"""
|
|
|
|
from app.core.sparse import SPARSE_DIM, SparseEncoder
|
|
|
|
|
|
class TestSparseEncoder:
|
|
"""SparseEncoder.encode / encode_batch 行为"""
|
|
|
|
def setup_method(self):
|
|
self.encoder = SparseEncoder()
|
|
|
|
def test_deterministic(self):
|
|
"""同一文本多次编码结果完全一致"""
|
|
text = "人工智能 artificial intelligence 2024"
|
|
assert self.encoder.encode(text) == self.encoder.encode(text)
|
|
|
|
def test_indices_sorted_unique(self):
|
|
"""indices 升序且无重复，与 values 等长，且都在 [0, SPARSE_DIM) 内"""
|
|
indices, values = self.encoder.encode("向量检索与关键词检索相结合 hybrid search 123")
|
|
assert indices == sorted(indices)
|
|
assert len(indices) == len(set(indices))
|
|
assert len(indices) == len(values)
|
|
assert all(0 <= idx < SPARSE_DIM for idx in indices)
|
|
assert all(v > 0 for v in values)
|
|
|
|
def test_mixed_text_non_empty(self):
|
|
"""中英文混合文本产生非空向量"""
|
|
indices, values = self.encoder.encode("使用 Qdrant 存储向量")
|
|
assert len(indices) > 0
|
|
assert len(values) > 0
|
|
|
|
def test_empty_string(self):
|
|
"""空字符串返回空向量"""
|
|
assert self.encoder.encode("") == ([], [])
|
|
|
|
def test_different_texts_differ(self):
|
|
"""不同文本产生不同向量"""
|
|
assert self.encoder.encode("机器学习") != self.encoder.encode("深度学习")
|
|
|
|
def test_english_case_insensitive(self):
|
|
"""英文词小写化：大小写不同编码结果一致"""
|
|
assert self.encoder.encode("Hello") == self.encoder.encode("hello")
|
|
|
|
def test_tf_weighting(self):
|
|
"""词频越高权重越大（1 + log(tf)）"""
|
|
once = dict(zip(*self.encoder.encode("apple"), strict=True))
|
|
twice = dict(zip(*self.encoder.encode("apple apple"), strict=True))
|
|
assert once.keys() == twice.keys()
|
|
for idx in once:
|
|
assert twice[idx] > once[idx]
|
|
|
|
def test_batch_matches_single(self):
|
|
"""encode_batch 与逐条 encode 结果一致"""
|
|
texts = ["人工智能", "vector search", ""]
|
|
batch = self.encoder.encode_batch(texts)
|
|
singles = [self.encoder.encode(t) for t in texts]
|
|
assert batch == singles
|