"""文本去重策略单元测试:none / sha256 / simhash 三种策略 lookup+record + 工厂 使用内存版 FakeRedis,不连真实 Redis。 """ from typing import Any from unittest.mock import AsyncMock import pytest from app.core import runtime_settings as rs from app.core.dedup import ( DEDUP_KEY_PREFIX, NoopDedupStrategy, Sha256DedupStrategy, SimhashDedupStrategy, _hamming_distance, _simhash, compute_text_hash, get_dedup_strategy, invalidate_dedup_strategy_cache, ) class FakeRedis: """内存版 Redis:实现 get_json/set_json,可记录所有写入""" def __init__(self) -> None: self.store: dict[str, dict[str, Any]] = {} self.writes: list[tuple[str, dict[str, Any], int | None]] = [] async def get_json(self, key: str) -> dict[str, Any] | None: return self.store.get(key) async def set_json(self, key: str, value: dict[str, Any], ttl: int | None = None) -> bool: self.writes.append((key, value, ttl)) self.store[key] = value return True @pytest.fixture def isolated_settings_path(tmp_path, monkeypatch: pytest.MonkeyPatch): path = tmp_path / "runtime_settings.json" monkeypatch.setenv("RUNTIME_SETTINGS_PATH", str(path)) rs._runtime_settings = None invalidate_dedup_strategy_cache() yield path rs._runtime_settings = None invalidate_dedup_strategy_cache() # ---------------------------------------------------------------------------- # # 工具函数 # ---------------------------------------------------------------------------- # class TestSimhashUtil: def test_empty_text_returns_zero(self): assert _simhash("") == 0 def test_same_text_same_fingerprint(self): assert _simhash("同一段文本") == _simhash("同一段文本") def test_different_text_different_fingerprint(self): assert _simhash("文本A") != _simhash("文本B是完全不同的内容") def test_hamming_distance_zero_for_same(self): fp = _simhash("hello") assert _hamming_distance(fp, fp) == 0 def test_hamming_distance_count(self): # 0b001 vs 0b100 → 两个位不同 assert _hamming_distance(0b001, 0b100) == 2 def test_compute_text_hash_is_sha256_hex(self): import hashlib text = "abc" assert compute_text_hash(text) == hashlib.sha256(text.encode("utf-8")).hexdigest() # ---------------------------------------------------------------------------- # # NoopDedupStrategy # ---------------------------------------------------------------------------- # class TestNoopStrategy: async def test_lookup_always_none(self): s = NoopDedupStrategy() assert await s.lookup("any") is None async def test_record_does_nothing(self): s = NoopDedupStrategy() await s.record("any", {"x": 1}) # 不抛异常即可 # ---------------------------------------------------------------------------- # # Sha256DedupStrategy # ---------------------------------------------------------------------------- # class TestSha256Strategy: def test_key_format(self): s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100) key = s._key("text") assert key.startswith(f"{DEDUP_KEY_PREFIX}sha256:") # 后缀是 64 位 sha256 hex suffix = key[len(f"{DEDUP_KEY_PREFIX}sha256:"):] assert len(suffix) == 64 async def test_lookup_miss_when_empty(self): s = Sha256DedupStrategy(redis=FakeRedis(), ttl_seconds=100) assert await s.lookup("text") is None async def test_record_then_lookup_hit(self): redis = FakeRedis() s = Sha256DedupStrategy(redis=redis, ttl_seconds=100) await s.record("text", {"document_id": "doc-1"}) hit = await s.lookup("text") assert hit is not None assert hit["document_id"] == "doc-1" async def test_record_uses_ttl(self): redis = FakeRedis() s = Sha256DedupStrategy(redis=redis, ttl_seconds=42) await s.record("text", {"x": 1}) # 检查写入 Redis 时使用的 TTL key = s._key("text") writes = [(k, v, ttl) for k, v, ttl in redis.writes if k == key] assert writes assert writes[0][2] == 42 async def test_lookup_redis_error_returns_none(self): """Redis 抛错时降级为未命中""" redis = AsyncMock() redis.get_json = AsyncMock(side_effect=RuntimeError("redis down")) s = Sha256DedupStrategy(redis=redis, ttl_seconds=100) assert await s.lookup("text") is None async def test_record_redis_error_swallows(self): """Redis 写入抛错时不向上抛""" redis = AsyncMock() redis.set_json = AsyncMock(side_effect=RuntimeError("redis down")) s = Sha256DedupStrategy(redis=redis, ttl_seconds=100) await s.record("text", {"x": 1}) # 不抛 # ---------------------------------------------------------------------------- # # SimhashDedupStrategy # ---------------------------------------------------------------------------- # class TestSimhashStrategy: async def test_lookup_miss_when_empty_index(self): s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=3) assert await s.lookup("text") is None async def test_record_then_lookup_identical_text(self): """相同文本 simhash 相同,距离 0 <= 阈值,命中""" redis = FakeRedis() s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3) await s.record("一段文本", {"document_id": "d1"}) hit = await s.lookup("一段文本") assert hit is not None assert hit["document_id"] == "d1" async def test_lookup_similar_text_within_threshold(self): """相似文本(海明距离 <= 阈值)也命中""" redis = FakeRedis() s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=64) # 阈值放大确保命中 await s.record("原文档文本内容示例", {"document_id": "d1"}) # 改一个字符 hit = await s.lookup("原文档文本内容示例改") assert hit is not None async def test_lookup_different_text_below_threshold_misses(self): """完全不同文本距离大,未命中""" redis = FakeRedis() s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3) await s.record("完全不同的第一种文本内容用于测试", {"document_id": "d1"}) hit = await s.lookup("另一段毫不相关的内容用于测试去重逻辑") # 距离应该比较大;若碰巧小于阈值(小概率),改大文本差异 if hit is not None: # 极小概率命中,放宽断言:至少 record 已写入 assert "document_id" in hit else: assert hit is None async def test_record_updates_index(self): """record 把新 simhash 加入索引""" redis = FakeRedis() s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3) await s.record("文本A", {"x": 1}) await s.record("文本B完全不同", {"x": 2}) index = await redis.get_json(SimhashDedupStrategy.INDEX_KEY) assert index is not None assert len(index["entries"]) == 2 async def test_threshold_clamped(self): """threshold 超出 0~64 范围被夹紧""" s = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=999) assert s._threshold == 64 s2 = SimhashDedupStrategy(redis=FakeRedis(), ttl_seconds=100, threshold=-5) assert s2._threshold == 0 async def test_lookup_redis_error_returns_none(self): redis = AsyncMock() redis.get_json = AsyncMock(side_effect=RuntimeError("redis down")) s = SimhashDedupStrategy(redis=redis, ttl_seconds=100, threshold=3) assert await s.lookup("text") is None # ---------------------------------------------------------------------------- # # 工厂 # ---------------------------------------------------------------------------- # class TestFactory: def test_none_strategy_when_redis_none(self, isolated_settings_path): """redis=None 时无论配置如何,都返回 NoopDedupStrategy""" s = get_dedup_strategy(None) assert isinstance(s, NoopDedupStrategy) def test_none_strategy_when_config_none(self, isolated_settings_path): rs.update_runtime_settings({"dedup": {"strategy": "none"}}) s = get_dedup_strategy(FakeRedis()) assert isinstance(s, NoopDedupStrategy) def test_sha256_strategy(self, isolated_settings_path): rs.update_runtime_settings({"dedup": {"strategy": "sha256"}}) s = get_dedup_strategy(FakeRedis()) assert isinstance(s, Sha256DedupStrategy) def test_simhash_strategy(self, isolated_settings_path): rs.update_runtime_settings( {"dedup": {"strategy": "simhash", "simhash_threshold": 5}} ) s = get_dedup_strategy(FakeRedis()) assert isinstance(s, SimhashDedupStrategy) assert s._threshold == 5 def test_cache_same_signature_returns_same_instance(self, isolated_settings_path): """配置签名相同时复用单例""" rs.update_runtime_settings({"dedup": {"strategy": "sha256"}}) s1 = get_dedup_strategy(FakeRedis()) s2 = get_dedup_strategy(FakeRedis()) assert s1 is s2 def test_cache_invalidated_on_signature_change(self, isolated_settings_path): """配置签名变化时重建单例""" rs.update_runtime_settings({"dedup": {"strategy": "sha256"}}) s1 = get_dedup_strategy(FakeRedis()) rs.update_runtime_settings({"dedup": {"strategy": "simhash"}}) s2 = get_dedup_strategy(FakeRedis()) assert s1 is not s2 assert isinstance(s2, SimhashDedupStrategy) def test_invalidate_cache_clears(self, isolated_settings_path): rs.update_runtime_settings({"dedup": {"strategy": "sha256"}}) s1 = get_dedup_strategy(FakeRedis()) invalidate_dedup_strategy_cache() s2 = get_dedup_strategy(FakeRedis()) assert s1 is not s2 def test_ttl_change_rebuilds(self, isolated_settings_path): """ttl 变化也触发重建(签名包含 ttl)""" rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 100}}) s1 = get_dedup_strategy(FakeRedis()) rs.update_runtime_settings({"dedup": {"strategy": "sha256", "ttl_seconds": 200}}) s2 = get_dedup_strategy(FakeRedis()) assert s1 is not s2