Initial commit: QMDSearch 分层信息检索服务

- FastAPI + Qdrant + Redis + Ollama 技术栈 - L1→L2→L3→chunk 四层分层检索（dense + sparse RRF 融合） - 文档三级总结与 2.5 级回退 - query 解析路由与分类 - /admin 管理页面
2026-07-29 21:24:40 +08:00
commit 51dc8dc4f6
83 changed files with 10794 additions and 0 deletions
@@ -0,0 +1,111 @@
+"""文档 API：POST /api/v1/documents 异步入库 + 任务查询 + 文档管理（列表/详情/删除）"""
+
+from typing import Any
+
+import structlog
+from fastapi import APIRouter, Query
+from fastapi.responses import JSONResponse
+
+from app.api.response import ApiError, ok
+from app.config import Settings
+from app.core.ingest_tasks import IngestTaskManager
+from app.core.ingestion import Ingester
+from app.models.document import DocumentInput
+from app.services.qdrant import QdrantService
+from app.services.redis import RedisCache, get_cache
+
+logger = structlog.get_logger()
+
+router = APIRouter(prefix="/api/v1", tags=["document"])
+
+# 模块级懒加载单例，避免每请求重建 Ingester 及其下游依赖
+_ingester: Ingester | None = None
+_qdrant: QdrantService | None = None
+_task_manager: IngestTaskManager | None = None
+
+
+def _get_ingester() -> Ingester:
+    global _ingester
+    if _ingester is None:
+        _ingester = Ingester()
+    return _ingester
+
+
+def _get_qdrant() -> QdrantService:
+    global _qdrant
+    if _qdrant is None:
+        _qdrant = QdrantService()
+    return _qdrant
+
+
+def _get_task_manager() -> IngestTaskManager:
+    """入库任务管理器懒加载单例
+
+    Redis 沿用全局缓存单例（RedisCache 读写全容错，不可用时镜像写失败仅告警）；
+    获取缓存实例异常时传 None，退化为纯内存模式。
+    """
+    global _task_manager
+    if _task_manager is None:
+        try:
+            redis: RedisCache | None = get_cache()
+        except Exception:
+            logger.warning("Redis 缓存不可用，入库任务状态仅保留在内存", exc_info=True)
+            redis = None
+        _task_manager = IngestTaskManager(ingester=_get_ingester(), redis=redis, settings=Settings())
+    return _task_manager
+
+
+@router.post("/documents")
+async def ingest_document(doc: DocumentInput) -> JSONResponse:
+    """文档入库入口：登记异步任务并返回 202 + task_id，入库结果经任务查询端点获取"""
+    if not doc.text.strip():
+        raise ApiError(1001, "文档内容不能为空")
+    task_id = await _get_task_manager().submit(doc)
+    return JSONResponse(status_code=202, content=ok({"task_id": task_id, "status": "pending"}))
+
+
+@router.get("/documents/tasks/{task_id}")
+async def get_ingest_task(task_id: str) -> dict[str, Any]:
+    """查询入库任务状态：含 task_id/status/created_at/updated_at，done 附 result，failed 附 error"""
+    task = await _get_task_manager().get(task_id)
+    if task is None:
+        raise ApiError(1004, "任务不存在")
+    return ok(task)
+
+
+@router.get("/documents")
+async def list_documents(
+    limit: int = Query(default=20, ge=1, le=100),
+    offset: str | None = None,
+) -> dict[str, Any]:
+    """分页列出文档（L1 摘要），返回 items 与下一页游标 next_offset"""
+    try:
+        items, next_offset = await _get_qdrant().scroll_l1(limit=limit, offset=offset)
+    except Exception as exc:
+        logger.error("文档列表查询失败", error=str(exc))
+        raise ApiError(2000, f"文档列表查询失败: {exc}") from exc
+    return ok({"items": items, "next_offset": next_offset})
+
+
+@router.get("/documents/{doc_id}")
+async def get_document(doc_id: str) -> dict[str, Any]:
+    """获取文档详情：L1 记录 + L2/L3 节点 + chunks 数量"""
+    try:
+        detail = await _get_qdrant().get_doc_detail(doc_id)
+    except Exception as exc:
+        logger.error("文档详情查询失败", doc_id=doc_id, error=str(exc))
+        raise ApiError(2000, f"文档详情查询失败: {exc}") from exc
+    if detail is None:
+        raise ApiError(1004, "文档不存在")
+    return ok(detail)
+
+
+@router.delete("/documents/{doc_id}")
+async def delete_document(doc_id: str) -> dict[str, Any]:
+    """删除文档：四层集合中该 doc_id 的所有点；幂等，不存在也返回成功（删除数全 0）"""
+    try:
+        deleted = await _get_qdrant().delete_by_doc_id(doc_id)
+    except Exception as exc:
+        logger.error("文档删除失败", doc_id=doc_id, error=str(exc))
+        raise ApiError(2000, f"文档删除失败: {exc}") from exc
+    return ok({"doc_id": doc_id, "deleted": deleted, "deleted_total": sum(deleted.values())})
@@ -0,0 +1,75 @@
+"""知识分类 API：GET /api/v1/knowledge/categories、GET /api/v1/knowledge/stats"""
+
+from functools import lru_cache
+from typing import Any
+
+import structlog
+from fastapi import APIRouter
+
+from app.api.response import ApiError, ok
+from app.config import settings
+from app.models.knowledge import UNCATEGORIZED, TaxonomyCategory, load_taxonomy
+from app.services.qdrant import ALL_COLLECTIONS, COLLECTION_L1, QdrantService
+
+logger = structlog.get_logger()
+
+router = APIRouter(prefix="/api/v1", tags=["knowledge"])
+
+# 类目分布统计的分页大小
+_STATS_SCROLL_PAGE_SIZE = 100
+
+# 模块级懒加载单例，避免每请求重建 QdrantService
+_qdrant: QdrantService | None = None
+
+
+def _get_qdrant() -> QdrantService:
+    global _qdrant
+    if _qdrant is None:
+        _qdrant = QdrantService()
+    return _qdrant
+
+
+@lru_cache(maxsize=1)
+def _get_taxonomy() -> list[TaxonomyCategory]:
+    """加载并缓存 taxonomy 类目集（进程内只加载一次）"""
+    return load_taxonomy(settings.taxonomy_path)
+
+
+@router.get("/knowledge/categories")
+async def list_categories() -> dict[str, Any]:
+    """返回完整知识分类类目集"""
+    categories = _get_taxonomy()
+    return ok({"categories": [c.model_dump() for c in categories], "count": len(categories)})
+
+
+@router.get("/knowledge/stats")
+async def knowledge_stats() -> dict[str, Any]:
+    """返回四层集合规模与 L1 类目分布统计"""
+    service = _get_qdrant()
+    try:
+        collections = {collection: await service.count(collection) for collection in ALL_COLLECTIONS}
+        categories = await _aggregate_l1_categories(service)
+    except Exception as exc:
+        logger.error("获取知识库统计失败", error=str(exc))
+        raise ApiError(2000, f"获取知识库统计失败: {exc}") from exc
+    return ok(
+        {
+            "collections": collections,
+            "categories": categories,
+            "uncategorized_count": categories.get(UNCATEGORIZED, 0),
+            "documents_total": collections[COLLECTION_L1],
+        }
+    )
+
+
+async def _aggregate_l1_categories(service: QdrantService) -> dict[str, int]:
+    """分页遍历 L1 文档，按 category 聚合文档数（空类目归入 uncategorized）"""
+    categories: dict[str, int] = {}
+    offset: str | None = None
+    while True:
+        items, offset = await service.scroll_l1(limit=_STATS_SCROLL_PAGE_SIZE, offset=offset)
+        for item in items:
+            category = item.get("category") or UNCATEGORIZED
+            categories[category] = categories.get(category, 0) + 1
+        if offset is None:
+            return categories
@@ -0,0 +1,58 @@
+"""检索 API：POST /api/v1/search"""
+
+from hashlib import sha256
+from typing import Any
+
+import structlog
+from fastapi import APIRouter
+
+from app.api.response import ApiError, ok
+from app.core.retriever import Retriever
+from app.models.search import SearchRequest
+from app.services.redis import get_cache
+
+logger = structlog.get_logger()
+
+router = APIRouter(prefix="/api/v1", tags=["search"])
+
+# 模块级懒加载单例，避免每请求重建 Qdrant/Embedding client
+_retriever: Retriever | None = None
+
+
+def _get_retriever() -> Retriever:
+    global _retriever
+    if _retriever is None:
+        _retriever = Retriever()
+    return _retriever
+
+
+def _cache_key(request: SearchRequest) -> str:
+    """检索缓存键：query + top_k 的短哈希（top_k 影响结果集，需参与键计算）"""
+    digest = sha256((request.query + "|" + str(request.top_k)).encode()).hexdigest()[:16]
+    return f"search:{digest}"
+
+
+@router.post("/search")
+async def search(request: SearchRequest) -> dict[str, Any]:
+    """分层检索入口，返回统一包装的 SearchResponse
+
+    先查 Redis 缓存：命中直接返回缓存的响应；未命中走检索流程并回写缓存。
+    缓存读写失败均降级为无缓存行为，不影响检索。
+    """
+    cache_key = _cache_key(request)
+    cached = await get_cache().get_json(cache_key)
+    if cached is not None:
+        logger.info("检索缓存命中", query=request.query, cache_key=cache_key)
+        return cached
+
+    try:
+        response = await _get_retriever().search(request)
+    except ApiError:
+        raise
+    except Exception as exc:
+        logger.exception("检索失败", query=request.query)
+        raise ApiError(2000, f"检索失败: {exc}") from exc
+
+    result = ok(response.model_dump())
+    await get_cache().set_json(cache_key, result)
+    return result