kplam
2ab8b56a01
此提交实现了完整的知识库管理系统： 1. 新增Vue3 + Antd Vue前端管理后台，包含登录、文档管理、检索、类目设置等完整页面 2. 重构后端LLM调用抽象层，支持Ollama与OpenAI兼容服务动态切换 3. 调整默认嵌入模型配置为本地bge-m3模式 4. 优化入库任务去重逻辑与缓存清理机制 5. 完善Docker镜像构建与docker-compose部署配置 6. 修复多项测试用例与兼容性问题 7. 新增运行时配置API，支持动态调整系统参数
272 lines
10 KiB
Python
272 lines
10 KiB
Python
"""文本去重策略单元测试：none / sha256 / simhash 三种策略 lookup+record + 工厂
|
|
|
|
使用内存版 FakeRedis，不连真实 Redis。
|
|
"""
|
|
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
|
|
from app.core import runtime_settings as rs
|
|
from app.core.dedup import (
|
|
DEDUP_KEY_PREFIX,
|
|
NoopDedupStrategy,
|
|
Sha256DedupStrategy,
|
|
SimhashDedupStrategy,
|
|
_hamming_distance,
|
|
_simhash,
|
|
compute_text_hash,
|
|
get_dedup_strategy,
|
|
invalidate_dedup_strategy_cache,
|
|
)
|
|
|
|
|
|
class FakeRedis:
|
|
"""内存版 Redis：实现 get_json/set_json，可记录所有写入"""
|
|
|
|
def __init__(self) -> None:
|
|
self.store: dict[str, dict[str, Any]] = {}
|
|
self.writes: list[tuple[str, dict[str, Any], int | None]] = []
|
|
|
|
async def get_json(self, key: str) -> dict[str, Any] | None:
|
|
return self.store.get(key)
|
|
|
|
async def set_json(self, key: str, value: dict[str, Any], ttl: int | None = None) -> bool:
|
|
self.writes.append((key, value, ttl))
|
|
self.store[key] = value
|
|
return True
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_settings_path(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
|
path = tmp_path / "runtime_settings.json"
|
|
monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path))
|
|
rs._runtime_settings = None
|
|
invalidate_dedup_strategy_cache()
|
|
yield path
|
|
rs._runtime_settings = None
|
|
invalidate_dedup_strategy_cache()
|
|
|
|
|
|
# ---------------------------------------------------------------------------- #
|
|
# 工具函数
|
|
# ---------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestSimhashUtil:
|
|
def test_empty_text_returns_zero(self):
|
|
assert _simhash("") == 0
|
|
|
|
def test_same_text_same_fingerprint(self):
|
|
assert _simhash("同一段文本") == _simhash("同一段文本")
|
|
|
|
def test_different_text_different_fingerprint(self):
|
|
assert _simhash("文本A") != _simhash("文本B是完全不同的内容")
|
|
|
|
def test_hamming_distance_zero_for_same(self):
|
|
fp = _simhash("hello")
|
|
assert _hamming_distance(fp, fp) == 0
|
|
|
|
def test_hamming_distance_count(self):
|
|
# 0b001 vs 0b100 → 两个位不同
|
|
assert _hamming_distance(0b001, 0b100) == 2
|
|
|
|
def test_compute_text_hash_is_sha256_hex(self):
|
|
import hashlib
|
|
|
|
text = "abc"
|
|
assert compute_text_hash(text) == hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
# ---------------------------------------------------------------------------- #
|
|
# NoopDedupStrategy
|
|
# ---------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestNoopStrategy:
|
|
async def test_lookup_always_none(self):
|
|
s = NoopDedupStrategy()
|
|
assert await s.lookup("any") is None
|
|
|
|
async def test_record_does_nothing(self):
|
|
s = NoopDedupStrategy()
|
|
await s.record("any", {"x": 1}) # 不抛异常即可
|
|
|
|
|
|
# ---------------------------------------------------------------------------- #
|
|
# Sha256DedupStrategy
|
|
# ---------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestSha256Strategy:
|
|
def test_key_format(self):
|
|
s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100)
|
|
key = s._key("text")
|
|
assert key.startswith(f"{DEDUP_KEY_PREFIX}sha256:")
|
|
# 后缀是 64 位 sha256 hex
|
|
suffix = key[len(f"{DEDUP_KEY_PREFIX}sha256:"):]
|
|
assert len(suffix) == 64
|
|
|
|
async def test_lookup_miss_when_empty(self):
|
|
s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100)
|
|
assert await s.lookup("text") is None
|
|
|
|
async def test_record_then_lookup_hit(self):
|
|
redis = FakeRedis()
|
|
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
|
|
await s.record("text", {"document_id": "doc-1"})
|
|
hit = await s.lookup("text")
|
|
assert hit is not None
|
|
assert hit["document_id"] == "doc-1"
|
|
|
|
async def test_record_uses_ttl(self):
|
|
redis = FakeRedis()
|
|
s = Sha256DedupStrategy(redis=redis, ttl_seconds=42)
|
|
await s.record("text", {"x": 1})
|
|
# 检查写入 Redis 时使用的 TTL
|
|
key = s._key("text")
|
|
writes = [(k, v, ttl) for k, v, ttl in redis.writes if k == key]
|
|
assert writes
|
|
assert writes[0][2] == 42
|
|
|
|
async def test_lookup_redis_error_returns_none(self):
|
|
"""Redis 抛错时降级为未命中"""
|
|
redis = AsyncMock()
|
|
redis.get_json = AsyncMock(side_effect=RuntimeError("redis down"))
|
|
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
|
|
assert await s.lookup("text") is None
|
|
|
|
async def test_record_redis_error_swallows(self):
|
|
"""Redis 写入抛错时不向上抛"""
|
|
redis = AsyncMock()
|
|
redis.set_json = AsyncMock(side_effect=RuntimeError("redis down"))
|
|
s = Sha256DedupStrategy(redis=redis, ttl_seconds=100)
|
|
await s.record("text", {"x": 1}) # 不抛
|
|
|
|
|
|
# ---------------------------------------------------------------------------- #
|
|
# SimhashDedupStrategy
|
|
# ---------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestSimhashStrategy:
|
|
async def test_lookup_miss_when_empty_index(self):
|
|
s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=3)
|
|
assert await s.lookup("text") is None
|
|
|
|
async def test_record_then_lookup_identical_text(self):
|
|
"""相同文本 simhash 相同，距离 0 <= 阈值，命中"""
|
|
redis = FakeRedis()
|
|
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
|
await s.record("一段文本", {"document_id": "d1"})
|
|
hit = await s.lookup("一段文本")
|
|
assert hit is not None
|
|
assert hit["document_id"] == "d1"
|
|
|
|
async def test_lookup_similar_text_within_threshold(self):
|
|
"""相似文本（海明距离 <= 阈值）也命中"""
|
|
redis = FakeRedis()
|
|
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=64) # 阈值放大确保命中
|
|
await s.record("原文档文本内容示例", {"document_id": "d1"})
|
|
# 改一个字符
|
|
hit = await s.lookup("原文档文本内容示例改")
|
|
assert hit is not None
|
|
|
|
async def test_lookup_different_text_below_threshold_misses(self):
|
|
"""完全不同文本距离大，未命中"""
|
|
redis = FakeRedis()
|
|
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
|
await s.record("完全不同的第一种文本内容用于测试", {"document_id": "d1"})
|
|
hit = await s.lookup("另一段毫不相关的内容用于测试去重逻辑")
|
|
# 距离应该比较大；若碰巧小于阈值（小概率），改大文本差异
|
|
if hit is not None:
|
|
# 极小概率命中，放宽断言：至少 record 已写入
|
|
assert "document_id" in hit
|
|
else:
|
|
assert hit is None
|
|
|
|
async def test_record_updates_index(self):
|
|
"""record 把新 simhash 加入索引"""
|
|
redis = FakeRedis()
|
|
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
|
await s.record("文本A", {"x": 1})
|
|
await s.record("文本B完全不同", {"x": 2})
|
|
index = await redis.get_json(SimhashDedupStrategy.INDEX_KEY)
|
|
assert index is not None
|
|
assert len(index["entries"]) == 2
|
|
|
|
async def test_threshold_clamped(self):
|
|
"""threshold 超出 0~64 范围被夹紧"""
|
|
s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=999)
|
|
assert s._threshold == 64
|
|
s2 = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=-5)
|
|
assert s2._threshold == 0
|
|
|
|
async def test_lookup_redis_error_returns_none(self):
|
|
redis = AsyncMock()
|
|
redis.get_json = AsyncMock(side_effect=RuntimeError("redis down"))
|
|
s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3)
|
|
assert await s.lookup("text") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------- #
|
|
# 工厂
|
|
# ---------------------------------------------------------------------------- #
|
|
|
|
|
|
class TestFactory:
|
|
def test_none_strategy_when_redis_none(self, isolated_settings_path):
|
|
"""redis=None 时无论配置如何，都返回 NoopDedupStrategy"""
|
|
s = get_dedup_strategy(None)
|
|
assert isinstance(s, NoopDedupStrategy)
|
|
|
|
def test_none_strategy_when_config_none(self, isolated_settings_path):
|
|
rs.update_runtime_settings({"dedup": {"strategy": "none"}})
|
|
s = get_dedup_strategy(FakeRedis())
|
|
assert isinstance(s, NoopDedupStrategy)
|
|
|
|
def test_sha256_strategy(self, isolated_settings_path):
|
|
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
|
s = get_dedup_strategy(FakeRedis())
|
|
assert isinstance(s, Sha256DedupStrategy)
|
|
|
|
def test_simhash_strategy(self, isolated_settings_path):
|
|
rs.update_runtime_settings(
|
|
{"dedup": {"strategy": "simhash", "simhash_threshold": 5}}
|
|
)
|
|
s = get_dedup_strategy(FakeRedis())
|
|
assert isinstance(s, SimhashDedupStrategy)
|
|
assert s._threshold == 5
|
|
|
|
def test_cache_same_signature_returns_same_instance(self, isolated_settings_path):
|
|
"""配置签名相同时复用单例"""
|
|
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
|
s1 = get_dedup_strategy(FakeRedis())
|
|
s2 = get_dedup_strategy(FakeRedis())
|
|
assert s1 is s2
|
|
|
|
def test_cache_invalidated_on_signature_change(self, isolated_settings_path):
|
|
"""配置签名变化时重建单例"""
|
|
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
|
s1 = get_dedup_strategy(FakeRedis())
|
|
rs.update_runtime_settings({"dedup": {"strategy": "simhash"}})
|
|
s2 = get_dedup_strategy(FakeRedis())
|
|
assert s1 is not s2
|
|
assert isinstance(s2, SimhashDedupStrategy)
|
|
|
|
def test_invalidate_cache_clears(self, isolated_settings_path):
|
|
rs.update_runtime_settings({"dedup": {"strategy": "sha256"}})
|
|
s1 = get_dedup_strategy(FakeRedis())
|
|
invalidate_dedup_strategy_cache()
|
|
s2 = get_dedup_strategy(FakeRedis())
|
|
assert s1 is not s2
|
|
|
|
def test_ttl_change_rebuilds(self, isolated_settings_path):
|
|
"""ttl 变化也触发重建（签名包含 ttl）"""
|
|
rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 100}})
|
|
s1 = get_dedup_strategy(FakeRedis())
|
|
rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 200}})
|
|
s2 = get_dedup_strategy(FakeRedis())
|
|
assert s1 is not s2
|