feat: 完成全量功能开发，包括前端管理后台与后端服务优化
此提交实现了完整的知识库管理系统： 1. 新增Vue3 + Antd Vue前端管理后台，包含登录、文档管理、检索、类目设置等完整页面 2. 重构后端LLM调用抽象层，支持Ollama与OpenAI兼容服务动态切换 3. 调整默认嵌入模型配置为本地bge-m3模式 4. 优化入库任务去重逻辑与缓存清理机制 5. 完善Docker镜像构建与docker-compose部署配置 6. 修复多项测试用例与兼容性问题 7. 新增运行时配置API，支持动态调整系统参数
This commit is contained in:
@@ -0,0 +1,216 @@
|
||||
"""文本去重策略工厂
|
||||
|
||||
支持三种策略：
|
||||
- none：关闭去重，每次都跑完整流水线
|
||||
- sha256：精确匹配（hash 完全相同才算重复）
|
||||
- simhash：近似匹配（海明距离 <= 阈值视为重复）
|
||||
|
||||
每种策略实现 DedupStrategy 协议：lookup(text) -> dict|None、record(text, result) -> None。
|
||||
工厂 get_dedup_strategy() 根据 runtime_settings.dedup 选择实现。
|
||||
所有策略共享同一 Redis key 前缀，但 key 后缀按策略区分避免冲突。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
import structlog
|
||||
|
||||
from app.core.runtime_settings import get_runtime_settings
|
||||
from app.services.redis import RedisCache
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# Redis 去重 key 前缀，按策略分桶
|
||||
_DEDUP_KEY_PREFIX = "dedup:"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DedupStrategy(Protocol):
|
||||
"""去重策略协议"""
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
"""查询文本是否已入库；命中返回旧 IngestionResult（dict），未命中返回 None"""
|
||||
...
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
"""记录文本已入库，供后续命中复用"""
|
||||
...
|
||||
|
||||
|
||||
class NoopDedupStrategy:
|
||||
"""关闭去重：永远不命中，也不记录"""
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class Sha256DedupStrategy:
|
||||
"""SHA256 精确去重：完全相同文本才算重复"""
|
||||
|
||||
def __init__(self, redis: RedisCache, ttl_seconds: int) -> None:
|
||||
self._redis = redis
|
||||
self._ttl = ttl_seconds
|
||||
|
||||
def _key(self, text: str) -> str:
|
||||
h = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
return f"{_DEDUP_KEY_PREFIX}sha256:{h}"
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
return await self._redis.get_json(self._key(text))
|
||||
except Exception:
|
||||
logger.warning("sha256 去重查询失败，降级未命中", exc_info=True)
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._redis.set_json(self._key(text), result_dict, ttl=self._ttl)
|
||||
except Exception:
|
||||
logger.warning("sha256 去重记录写入失败", exc_info=True)
|
||||
|
||||
|
||||
# SimHash 实现（64 位）
|
||||
_MASK_64 = (1 << 64) - 1
|
||||
|
||||
|
||||
def _simhash(text: str, token_size: int = 4) -> int:
|
||||
"""计算文本 64 位 simhash
|
||||
|
||||
简化实现：按 token_size 字符滑窗分词，每段 md5 → 128 位 → 取低 64 位作为 hash，
|
||||
逐位加权（hash 该位为 1 则 +1，为 0 则 -1），最终符号位定 1/0。
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
tokens = [text[i : i + token_size] for i in range(0, len(text), token_size)]
|
||||
weights = [1] * 64
|
||||
vec = [0] * 64
|
||||
for token in tokens:
|
||||
h = int(hashlib.md5(token.encode("utf-8")).hexdigest(), 16) & _MASK_64
|
||||
for i in range(64):
|
||||
bit = (h >> i) & 1
|
||||
vec[i] += weights[i] if bit else -weights[i]
|
||||
fingerprint = 0
|
||||
for i in range(64):
|
||||
if vec[i] > 0:
|
||||
fingerprint |= (1 << i)
|
||||
return fingerprint
|
||||
|
||||
|
||||
def _hamming_distance(a: int, b: int) -> int:
|
||||
return bin((a ^ b) & _MASK_64).count("1")
|
||||
|
||||
|
||||
class SimhashDedupStrategy:
|
||||
"""SimHash 近似去重：海明距离 <= 阈值视为重复
|
||||
|
||||
Redis 中存所有已入库文本的 simhash（value 为 IngestionResult + simhash）。
|
||||
lookup 时遍历所有候选 simhash 算海明距离，找最近的一个 <= 阈值则命中。
|
||||
注：本实现为简化版，全表扫描，适合中小规模知识库；超大规模需换 LSH 索引。
|
||||
"""
|
||||
|
||||
INDEX_KEY = f"{_DEDUP_KEY_PREFIX}simhash:index" # list 形式存所有 simhash+key
|
||||
|
||||
def __init__(self, redis: RedisCache, ttl_seconds: int, threshold: int = 3) -> None:
|
||||
self._redis = redis
|
||||
self._ttl = ttl_seconds
|
||||
self._threshold = max(0, min(64, threshold))
|
||||
|
||||
async def lookup(self, text: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
fingerprint = _simhash(text)
|
||||
# 简化：扫描所有 simhash 记录找最近的
|
||||
# 用 set:dedup:simhash:fps 存所有 fingerprint，每个 fp 对应一个 record key
|
||||
# 这里用 list key 简化（适合小规模）
|
||||
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
|
||||
entries = index.get("entries", [])
|
||||
best_dist = self._threshold + 1
|
||||
best_key: str | None = None
|
||||
for entry in entries:
|
||||
fp = entry.get("fingerprint")
|
||||
key = entry.get("key")
|
||||
if fp is None or key is None:
|
||||
continue
|
||||
dist = _hamming_distance(fingerprint, int(fp))
|
||||
if dist <= self._threshold and dist < best_dist:
|
||||
best_dist = dist
|
||||
best_key = key
|
||||
if best_key is None:
|
||||
return None
|
||||
return await self._redis.get_json(best_key)
|
||||
except Exception:
|
||||
logger.warning("simhash 去重查询失败，降级未命中", exc_info=True)
|
||||
return None
|
||||
|
||||
async def record(self, text: str, result_dict: dict[str, Any]) -> None:
|
||||
try:
|
||||
fingerprint = _simhash(text)
|
||||
key = f"{_DEDUP_KEY_PREFIX}simhash:{fingerprint:016x}"
|
||||
# 1. 写记录
|
||||
await self._redis.set_json(key, result_dict, ttl=self._ttl)
|
||||
# 2. 更新索引
|
||||
index = await self._redis.get_json(self.INDEX_KEY) or {"entries": []}
|
||||
entries = index.get("entries", [])
|
||||
entries.append({"fingerprint": fingerprint, "key": key})
|
||||
await self._redis.set_json(self.INDEX_KEY, {"entries": entries}, ttl=self._ttl)
|
||||
except Exception:
|
||||
logger.warning("simhash 去重记录写入失败", exc_info=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# 工厂
|
||||
# ---------------------------------------------------------------------------- #
|
||||
|
||||
_strategy_cache: DedupStrategy | None = None
|
||||
_strategy_signature: tuple[str, int, int] | None = None # (strategy, ttl, threshold)
|
||||
|
||||
|
||||
def get_dedup_strategy(redis: RedisCache | None) -> DedupStrategy:
|
||||
"""按 runtime_settings.dedup 构造去重策略
|
||||
|
||||
Redis 不可用时降级为 NoopDedupStrategy（关闭去重），不影响主流程。
|
||||
策略配置变更时自动重建单例。
|
||||
"""
|
||||
global _strategy_cache, _strategy_signature
|
||||
|
||||
rt = get_runtime_settings()
|
||||
sig = (rt.dedup.strategy, rt.dedup.ttl_seconds, rt.dedup.simhash_threshold)
|
||||
|
||||
if _strategy_cache is not None and _strategy_signature == sig:
|
||||
return _strategy_cache
|
||||
|
||||
if redis is None or rt.dedup.strategy == "none":
|
||||
strategy: DedupStrategy = NoopDedupStrategy()
|
||||
elif rt.dedup.strategy == "sha256":
|
||||
strategy = Sha256DedupStrategy(redis=redis, ttl_seconds=rt.dedup.ttl_seconds)
|
||||
elif rt.dedup.strategy == "simhash":
|
||||
strategy = SimhashDedupStrategy(
|
||||
redis=redis, ttl_seconds=rt.dedup.ttl_seconds, threshold=rt.dedup.simhash_threshold
|
||||
)
|
||||
else:
|
||||
logger.warning("未知去重策略，降级为 noop", requested=rt.dedup.strategy)
|
||||
strategy = NoopDedupStrategy()
|
||||
|
||||
_strategy_cache = strategy
|
||||
_strategy_signature = sig
|
||||
return strategy
|
||||
|
||||
|
||||
def invalidate_dedup_strategy_cache() -> None:
|
||||
"""清除策略缓存（runtime_settings 更新后调用）"""
|
||||
global _strategy_cache, _strategy_signature
|
||||
_strategy_cache = None
|
||||
_strategy_signature = None
|
||||
|
||||
|
||||
def compute_text_hash(text: str) -> str:
|
||||
"""计算文本哈希（兼容旧接口，sha256 策略下与 dedup key 一致）"""
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
# 兼容旧测试与外部引用：保留 DEDUP_KEY_PREFIX 导出
|
||||
DEDUP_KEY_PREFIX = _DEDUP_KEY_PREFIX
|
||||
Reference in New Issue
Block a user