feat: 新增多格式文件上传入库与认证体系
- 新增 JWT 认证模块,支持登录/注册/用户管理 - 新增文件上传接口,支持 .txt/.md/.html/.pdf/.docx 等格式解析入库 - 新增检索结果 AI 总结功能 - 新增文本去重缓存机制 - 新增全局认证夹具简化测试 - 新增配置项与环境变量支持 - 完善文档与测试覆盖
This commit is contained in:
+39
-14
@@ -15,11 +15,12 @@ from qdrant_client import models
|
||||
|
||||
from app.config import settings
|
||||
from app.core.embeddings import EmbeddingService, create_embedding_service
|
||||
from app.core.query_parser import QueryParser
|
||||
from app.core.query_parser import QueryParser, RouteDecision
|
||||
from app.core.ranker import finalize, rrf_fuse
|
||||
from app.core.result_summarizer import ResultSummarizer
|
||||
from app.core.sparse import SparseEncoder
|
||||
from app.models.knowledge import load_taxonomy
|
||||
from app.models.search import SearchHit, SearchRequest, SearchResponse
|
||||
from app.models.search import ExtractedInfo, SearchHit, SearchRequest, SearchResponse
|
||||
from app.services.ollama import OllamaClient
|
||||
from app.services.qdrant import (
|
||||
COLLECTION_CHUNKS,
|
||||
@@ -47,6 +48,7 @@ class Retriever:
|
||||
query_parser: QueryParser | None = None,
|
||||
embedding: EmbeddingService | None = None,
|
||||
sparse_encoder: SparseEncoder | None = None,
|
||||
result_summarizer: ResultSummarizer | None = None,
|
||||
) -> None:
|
||||
self.qdrant = qdrant or QdrantService()
|
||||
self.query_parser = query_parser or QueryParser(
|
||||
@@ -55,6 +57,7 @@ class Retriever:
|
||||
)
|
||||
self.embedding = embedding or create_embedding_service()
|
||||
self.sparse_encoder = sparse_encoder or SparseEncoder()
|
||||
self.result_summarizer = result_summarizer or ResultSummarizer()
|
||||
|
||||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||||
"""分层检索主流程"""
|
||||
@@ -74,12 +77,8 @@ class Retriever:
|
||||
# L1 无候选文档 → 全库 chunk 兜底
|
||||
chunk_hits = await self._search_collection(COLLECTION_CHUNKS, dense, sparse, settings.retrieval_top_k, None)
|
||||
logger.info("L1 无命中,全库 chunk 兜底", hits=len(chunk_hits))
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
hits=self._to_hits(finalize(chunk_hits, self._final_k(request))),
|
||||
routed_categories=route.filter_categories or [],
|
||||
fallback=True,
|
||||
)
|
||||
hits = self._to_hits(finalize(chunk_hits, self._final_k(request)))
|
||||
return await self._build_response(request, route, hits, fallback=True)
|
||||
|
||||
doc_ids = _unique((p.payload or {}).get("doc_id") for p in l1_hits)
|
||||
|
||||
@@ -125,12 +124,8 @@ class Retriever:
|
||||
logger.info("chunk 检索完成", hits=len(chunk_hits))
|
||||
|
||||
final_points = finalize(rrf_fuse([chunk_hits]), self._final_k(request))
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
hits=self._to_hits(final_points),
|
||||
routed_categories=route.filter_categories or [],
|
||||
fallback=route.fallback,
|
||||
)
|
||||
hits = self._to_hits(final_points)
|
||||
return await self._build_response(request, route, hits, fallback=route.fallback)
|
||||
|
||||
async def _search_collection(
|
||||
self,
|
||||
@@ -167,3 +162,33 @@ class Retriever:
|
||||
)
|
||||
)
|
||||
return hits
|
||||
|
||||
async def _build_response(
|
||||
self,
|
||||
request: SearchRequest,
|
||||
route: RouteDecision,
|
||||
hits: list[SearchHit],
|
||||
*,
|
||||
fallback: bool,
|
||||
) -> SearchResponse:
|
||||
"""构造最终响应:组装 AI 提取信息,按需生成结果总结"""
|
||||
parsed = route.parsed
|
||||
extracted = ExtractedInfo(
|
||||
rewrite=parsed.rewrite,
|
||||
keywords=parsed.keywords,
|
||||
entities=parsed.entities,
|
||||
intent=parsed.intent,
|
||||
time_range=parsed.time_range,
|
||||
categories=[c.name for c in parsed.categories],
|
||||
)
|
||||
summary: str | None = None
|
||||
if request.summarize:
|
||||
summary = await self.result_summarizer.summarize(request.query, hits)
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
hits=hits,
|
||||
routed_categories=route.filter_categories or [],
|
||||
fallback=fallback,
|
||||
extracted_info=extracted,
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user