feat: 新增多格式文件上传入库与认证体系

- 新增 JWT 认证模块,支持登录/注册/用户管理
- 新增文件上传接口,支持 .txt/.md/.html/.pdf/.docx 等格式解析入库
- 新增检索结果 AI 总结功能
- 新增文本去重缓存机制
- 新增全局认证夹具简化测试
- 新增配置项与环境变量支持
- 完善文档与测试覆盖
This commit is contained in:
2026-07-30 10:30:15 +08:00
parent 51dc8dc4f6
commit dce9e31bde
31 changed files with 3021 additions and 45 deletions
+39 -14
View File
@@ -15,11 +15,12 @@ from qdrant_client import models
from app.config import settings
from app.core.embeddings import EmbeddingService, create_embedding_service
from app.core.query_parser import QueryParser
from app.core.query_parser import QueryParser, RouteDecision
from app.core.ranker import finalize, rrf_fuse
from app.core.result_summarizer import ResultSummarizer
from app.core.sparse import SparseEncoder
from app.models.knowledge import load_taxonomy
from app.models.search import SearchHit, SearchRequest, SearchResponse
from app.models.search import ExtractedInfo, SearchHit, SearchRequest, SearchResponse
from app.services.ollama import OllamaClient
from app.services.qdrant import (
COLLECTION_CHUNKS,
@@ -47,6 +48,7 @@ class Retriever:
query_parser: QueryParser | None = None,
embedding: EmbeddingService | None = None,
sparse_encoder: SparseEncoder | None = None,
result_summarizer: ResultSummarizer | None = None,
) -> None:
self.qdrant = qdrant or QdrantService()
self.query_parser = query_parser or QueryParser(
@@ -55,6 +57,7 @@ class Retriever:
)
self.embedding = embedding or create_embedding_service()
self.sparse_encoder = sparse_encoder or SparseEncoder()
self.result_summarizer = result_summarizer or ResultSummarizer()
async def search(self, request: SearchRequest) -> SearchResponse:
"""分层检索主流程"""
@@ -74,12 +77,8 @@ class Retriever:
# L1 无候选文档 → 全库 chunk 兜底
chunk_hits = await self._search_collection(COLLECTION_CHUNKS, dense, sparse, settings.retrieval_top_k, None)
logger.info("L1 无命中,全库 chunk 兜底", hits=len(chunk_hits))
return SearchResponse(
query=request.query,
hits=self._to_hits(finalize(chunk_hits, self._final_k(request))),
routed_categories=route.filter_categories or [],
fallback=True,
)
hits = self._to_hits(finalize(chunk_hits, self._final_k(request)))
return await self._build_response(request, route, hits, fallback=True)
doc_ids = _unique((p.payload or {}).get("doc_id") for p in l1_hits)
@@ -125,12 +124,8 @@ class Retriever:
logger.info("chunk 检索完成", hits=len(chunk_hits))
final_points = finalize(rrf_fuse([chunk_hits]), self._final_k(request))
return SearchResponse(
query=request.query,
hits=self._to_hits(final_points),
routed_categories=route.filter_categories or [],
fallback=route.fallback,
)
hits = self._to_hits(final_points)
return await self._build_response(request, route, hits, fallback=route.fallback)
async def _search_collection(
self,
@@ -167,3 +162,33 @@ class Retriever:
)
)
return hits
async def _build_response(
self,
request: SearchRequest,
route: RouteDecision,
hits: list[SearchHit],
*,
fallback: bool,
) -> SearchResponse:
"""构造最终响应:组装 AI 提取信息,按需生成结果总结"""
parsed = route.parsed
extracted = ExtractedInfo(
rewrite=parsed.rewrite,
keywords=parsed.keywords,
entities=parsed.entities,
intent=parsed.intent,
time_range=parsed.time_range,
categories=[c.name for c in parsed.categories],
)
summary: str | None = None
if request.summarize:
summary = await self.result_summarizer.summarize(request.query, hits)
return SearchResponse(
query=request.query,
hits=hits,
routed_categories=route.filter_categories or [],
fallback=fallback,
extracted_info=extracted,
summary=summary,
)