kplam
51dc8dc4f6
- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
248 lines
9.1 KiB
Python
248 lines
9.1 KiB
Python
"""Redis 缓存与缓存集成测试（AsyncMock redis 客户端，不连真实 Redis）"""
|
||
|
||
import json
|
||
from hashlib import sha256
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock
|
||
|
||
import pytest
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.config import settings
|
||
from app.core.query_parser import CategoryHit, ParsedQuery, QueryParser
|
||
from app.main import app
|
||
from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory
|
||
from app.models.search import SearchRequest, SearchResponse
|
||
from app.services.redis import RedisCache
|
||
|
||
|
||
def _cache_with_client(client: AsyncMock) -> RedisCache:
|
||
"""构造注入 mock 客户端的 RedisCache（绕过真实 Redis 连接）"""
|
||
cache = RedisCache()
|
||
cache._client = client
|
||
return cache
|
||
|
||
|
||
class _MemoryCache:
|
||
"""内存版假缓存，接口与 RedisCache 一致"""
|
||
|
||
def __init__(self) -> None:
|
||
self.store: dict[str, dict[str, Any]] = {}
|
||
|
||
async def get_json(self, key: str) -> dict[str, Any] | None:
|
||
return self.store.get(key)
|
||
|
||
async def set_json(self, key: str, value: dict[str, Any], ttl: int | None = None) -> bool:
|
||
self.store[key] = value
|
||
return True
|
||
|
||
async def close(self) -> None:
|
||
pass
|
||
|
||
|
||
class TestRedisCacheGetJson:
|
||
"""RedisCache.get_json：命中 / 未命中 / 数据异常 / 连接异常"""
|
||
|
||
async def test_hit(self):
|
||
client = AsyncMock()
|
||
client.get.return_value = json.dumps({"code": 0, "data": {"x": 1}}, ensure_ascii=False)
|
||
cache = _cache_with_client(client)
|
||
|
||
assert await cache.get_json("k") == {"code": 0, "data": {"x": 1}}
|
||
client.get.assert_awaited_once_with("k")
|
||
|
||
async def test_miss(self):
|
||
client = AsyncMock()
|
||
client.get.return_value = None
|
||
|
||
assert await _cache_with_client(client).get_json("k") is None
|
||
|
||
async def test_invalid_json_returns_none(self):
|
||
client = AsyncMock()
|
||
client.get.return_value = "not-json{"
|
||
|
||
assert await _cache_with_client(client).get_json("k") is None
|
||
|
||
async def test_non_dict_json_returns_none(self):
|
||
client = AsyncMock()
|
||
client.get.return_value = json.dumps([1, 2, 3])
|
||
|
||
assert await _cache_with_client(client).get_json("k") is None
|
||
|
||
async def test_error_returns_none(self):
|
||
client = AsyncMock()
|
||
client.get.side_effect = ConnectionError("redis down")
|
||
|
||
assert await _cache_with_client(client).get_json("k") is None
|
||
|
||
|
||
class TestRedisCacheSetJson:
|
||
"""RedisCache.set_json：默认 TTL / 自定义 TTL / 异常容错"""
|
||
|
||
async def test_ok_uses_default_ttl(self):
|
||
client = AsyncMock()
|
||
cache = _cache_with_client(client)
|
||
|
||
assert await cache.set_json("k", {"a": 1}) is True
|
||
client.setex.assert_awaited_once_with("k", settings.cache_ttl, json.dumps({"a": 1}, ensure_ascii=False))
|
||
|
||
async def test_ok_custom_ttl(self):
|
||
client = AsyncMock()
|
||
cache = _cache_with_client(client)
|
||
|
||
assert await cache.set_json("k", {"a": 1}, ttl=10) is True
|
||
client.setex.assert_awaited_once_with("k", 10, json.dumps({"a": 1}, ensure_ascii=False))
|
||
|
||
async def test_error_returns_false(self):
|
||
client = AsyncMock()
|
||
client.setex.side_effect = ConnectionError("redis down")
|
||
|
||
assert await _cache_with_client(client).set_json("k", {"a": 1}) is False
|
||
|
||
|
||
class _CountingRetriever:
|
||
"""记录 search 调用次数的假 Retriever"""
|
||
|
||
def __init__(self, response: SearchResponse) -> None:
|
||
self.response = response
|
||
self.calls = 0
|
||
|
||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||
self.calls += 1
|
||
return self.response
|
||
|
||
|
||
def _search_cache_key(query: str, top_k: int | None = None) -> str:
|
||
"""与路由侧一致的检索缓存键"""
|
||
return f"search:{sha256((query + '|' + str(top_k)).encode()).hexdigest()[:16]}"
|
||
|
||
|
||
class TestSearchApiCache:
|
||
"""检索 API 缓存层：命中短路 Retriever，Redis 异常不影响检索"""
|
||
|
||
def test_second_request_hits_cache(self, monkeypatch: pytest.MonkeyPatch):
|
||
"""第一次调 Retriever 并回写缓存，第二次缓存命中不再调用"""
|
||
retriever = _CountingRetriever(SearchResponse(query="q"))
|
||
cache = _MemoryCache()
|
||
monkeypatch.setattr("app.api.v1.search._retriever", retriever)
|
||
monkeypatch.setattr("app.api.v1.search.get_cache", lambda: cache)
|
||
|
||
client = TestClient(app)
|
||
body1 = client.post("/api/v1/search", json={"query": "q"}).json()
|
||
body2 = client.post("/api/v1/search", json={"query": "q"}).json()
|
||
|
||
assert retriever.calls == 1
|
||
assert body1 == body2
|
||
assert body2["code"] == 0
|
||
assert _search_cache_key("q") in cache.store
|
||
|
||
def test_different_top_k_uses_different_key(self, monkeypatch: pytest.MonkeyPatch):
|
||
"""top_k 参与缓存键计算：同 query 不同 top_k 不共享缓存"""
|
||
retriever = _CountingRetriever(SearchResponse(query="q"))
|
||
cache = _MemoryCache()
|
||
monkeypatch.setattr("app.api.v1.search._retriever", retriever)
|
||
monkeypatch.setattr("app.api.v1.search.get_cache", lambda: cache)
|
||
|
||
client = TestClient(app)
|
||
client.post("/api/v1/search", json={"query": "q"})
|
||
client.post("/api/v1/search", json={"query": "q", "top_k": 3})
|
||
|
||
assert retriever.calls == 2
|
||
assert _search_cache_key("q") in cache.store
|
||
assert _search_cache_key("q", 3) in cache.store
|
||
|
||
def test_redis_error_still_returns_result(self, monkeypatch: pytest.MonkeyPatch):
|
||
"""Redis 读写全部抛异常 → 降级为无缓存，检索正常返回"""
|
||
broken_client = AsyncMock()
|
||
broken_client.get.side_effect = ConnectionError("redis down")
|
||
broken_client.setex.side_effect = ConnectionError("redis down")
|
||
retriever = _CountingRetriever(SearchResponse(query="q"))
|
||
monkeypatch.setattr("app.api.v1.search._retriever", retriever)
|
||
monkeypatch.setattr("app.api.v1.search.get_cache", lambda: _cache_with_client(broken_client))
|
||
|
||
client = TestClient(app)
|
||
resp = client.post("/api/v1/search", json={"query": "q"})
|
||
|
||
assert resp.status_code == 200
|
||
body = resp.json()
|
||
assert body["code"] == 0
|
||
assert body["data"]["query"] == "q"
|
||
assert retriever.calls == 1
|
||
|
||
|
||
def _taxonomy() -> list[TaxonomyCategory]:
|
||
return [
|
||
TaxonomyCategory(name="技术文档", description="技术资料"),
|
||
TaxonomyCategory(name=UNCATEGORIZED, description="未分类"),
|
||
]
|
||
|
||
|
||
class _FakeOllama:
|
||
"""返回固定响应的假 OllamaClient，记录调用次数"""
|
||
|
||
def __init__(self, response: str) -> None:
|
||
self.response = response
|
||
self.calls: list[str] = []
|
||
|
||
async def generate(self, prompt: str, json_mode: bool = False) -> str:
|
||
self.calls.append(prompt)
|
||
return self.response
|
||
|
||
|
||
def _qparse_cache_key(query: str) -> str:
|
||
"""与 QueryParser 一致的解析缓存键"""
|
||
return f"qparse:{sha256(query.encode()).hexdigest()[:16]}"
|
||
|
||
|
||
class TestQueryParserCache:
|
||
"""QueryParser 解析缓存：命中跳过 LLM，异常回退 LLM"""
|
||
|
||
async def test_cache_hit_skips_ollama(self):
|
||
"""缓存命中时直接用缓存重建 ParsedQuery，不调用 Ollama；decide_route 仍现算"""
|
||
parsed = ParsedQuery(
|
||
raw_query="q",
|
||
rewrite="缓存里的 rewrite",
|
||
keywords=["k"],
|
||
categories=[CategoryHit(name="技术文档", confidence=0.9)],
|
||
)
|
||
cache = _MemoryCache()
|
||
cache.store[_qparse_cache_key("q")] = parsed.model_dump()
|
||
ollama = _FakeOllama("不应被调用")
|
||
parser = QueryParser(ollama=ollama, taxonomy=_taxonomy(), cache=cache) # type: ignore[arg-type]
|
||
|
||
decision = await parser.parse_and_route("q")
|
||
|
||
assert ollama.calls == []
|
||
assert decision.parsed.rewrite == "缓存里的 rewrite"
|
||
assert decision.fallback is False
|
||
assert decision.reason == "routed"
|
||
assert decision.filter_categories == ["技术文档"]
|
||
|
||
async def test_llm_result_written_to_cache(self):
|
||
"""首次走 LLM 并回写缓存，第二次同 query 命中缓存不再调 LLM"""
|
||
ollama = _FakeOllama('{"categories": [], "rewrite": "r", "keywords": []}')
|
||
cache = _MemoryCache()
|
||
parser = QueryParser(ollama=ollama, taxonomy=_taxonomy(), cache=cache) # type: ignore[arg-type]
|
||
|
||
await parser.parse_and_route("q")
|
||
await parser.parse_and_route("q")
|
||
|
||
assert len(ollama.calls) == 1
|
||
assert _qparse_cache_key("q") in cache.store
|
||
|
||
async def test_cache_error_falls_back_to_llm(self):
|
||
"""Redis 读写全部抛异常 → 降级为无缓存，正常走 LLM 解析"""
|
||
broken_client = AsyncMock()
|
||
broken_client.get.side_effect = ConnectionError("redis down")
|
||
broken_client.setex.side_effect = ConnectionError("redis down")
|
||
ollama = _FakeOllama('{"categories": [], "rewrite": "r", "keywords": []}')
|
||
parser = QueryParser(
|
||
ollama=ollama, taxonomy=_taxonomy(), cache=_cache_with_client(broken_client) # type: ignore[arg-type]
|
||
)
|
||
|
||
decision = await parser.parse_and_route("q")
|
||
|
||
assert len(ollama.calls) == 1
|
||
assert decision.parsed.rewrite == "r"
|
||
assert decision.reason == "low_confidence"
|