kplam
bdd30f0a88
- deps.py: _create_redis_client 增加同步 ping 校验，Redis 不可达时正确降级为内存模式 - main.py: lifespan 复用 deps 的 UserStore 单例，避免 admin 与 API 请求实例不一致 - 前端: 强制改密弹窗（must_change_password 用户）、兼容新旧登录返回格式、markPasswordChanged - 新增 NAS SSH 部署脚本与批处理测试；gitignore 前端构建产物
405 lines
15 KiB
Python
405 lines
15 KiB
Python
"""文档 API：POST /api/v1/documents 异步入库 + 任务查询 + 文档管理（列表/详情/删除）+ 文件上传入库"""
|
||
|
||
import json
|
||
import uuid
|
||
from datetime import UTC, datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import structlog
|
||
from fastapi import APIRouter, Depends, File, Form, Query, UploadFile
|
||
from fastapi.responses import FileResponse, JSONResponse
|
||
|
||
from app.api.deps import get_current_user
|
||
from app.api.response import ApiError, ok
|
||
from app.config import Settings, settings
|
||
from app.core.file_parser import parse_file
|
||
from app.core.ingest_tasks import IngestTaskManager
|
||
from app.core.ingestion import Ingester
|
||
from app.core.users import UserRecord
|
||
from app.models.document import DocumentInput
|
||
from app.services.qdrant import QdrantService
|
||
from app.services.redis import RedisCache, get_cache
|
||
|
||
logger = structlog.get_logger()
|
||
|
||
router = APIRouter(prefix="/api/v1", tags=["document"])
|
||
|
||
# 模块级懒加载单例，避免每请求重建 Ingester 及其下游依赖
|
||
_ingester: Ingester | None = None
|
||
_qdrant: QdrantService | None = None
|
||
_task_manager: IngestTaskManager | None = None
|
||
|
||
|
||
def _get_ingester() -> Ingester:
|
||
global _ingester
|
||
if _ingester is None:
|
||
_ingester = Ingester()
|
||
return _ingester
|
||
|
||
|
||
def _get_qdrant() -> QdrantService:
|
||
global _qdrant
|
||
if _qdrant is None:
|
||
_qdrant = QdrantService()
|
||
return _qdrant
|
||
|
||
|
||
def _get_task_manager() -> IngestTaskManager:
|
||
"""入库任务管理器懒加载单例
|
||
|
||
Redis 沿用全局缓存单例（RedisCache 读写全容错，不可用时镜像写失败仅告警）；
|
||
获取缓存实例异常时传 None，退化为纯内存模式。
|
||
"""
|
||
global _task_manager
|
||
if _task_manager is None:
|
||
try:
|
||
redis: RedisCache | None = get_cache()
|
||
except Exception:
|
||
logger.warning("Redis 缓存不可用，入库任务状态仅保留在内存", exc_info=True)
|
||
redis = None
|
||
_task_manager = IngestTaskManager(
|
||
ingester=_get_ingester(), redis=redis, settings=Settings()
|
||
)
|
||
return _task_manager
|
||
|
||
|
||
def _allowed_extensions() -> set[str]:
|
||
"""解析 settings.upload_allowed_extensions 逗号分隔字符串为扩展名集合（全小写、含点号）"""
|
||
return {
|
||
ext.strip().lower()
|
||
for ext in settings.upload_allowed_extensions.split(",")
|
||
if ext.strip()
|
||
}
|
||
|
||
|
||
@router.post("/documents")
|
||
async def ingest_document(
|
||
doc: DocumentInput, user: UserRecord = Depends(get_current_user)
|
||
) -> JSONResponse:
|
||
"""文档入库入口：登记异步任务并返回 202 + task_id，入库结果经任务查询端点获取"""
|
||
if not doc.text.strip():
|
||
raise ApiError(1001, "文档内容不能为空")
|
||
task_id = await _get_task_manager().submit(doc)
|
||
return JSONResponse(
|
||
status_code=202, content=ok({"task_id": task_id, "status": "pending"})
|
||
)
|
||
|
||
|
||
@router.post("/documents/upload")
|
||
async def upload_document(
|
||
file: UploadFile = File(
|
||
..., description="上传的文件（.txt/.md/.html/.htm/.pdf/.docx）"
|
||
),
|
||
title: str = Form(default="", description="可选标题，默认取原文件名去扩展"),
|
||
source: str = Form(default="", description="可选来源标识，默认 file:{原文件名}"),
|
||
metadata: str = Form(
|
||
default="", description='可选元数据 JSON 字符串，如 \'{"author":"x"}\''
|
||
),
|
||
user: UserRecord = Depends(get_current_user),
|
||
) -> JSONResponse:
|
||
"""文件上传入库入口：校验 → 提取文本 → 落盘 → 提交异步入库流水线
|
||
|
||
与 POST /documents 共用同一 IngestTaskManager；任务状态经
|
||
GET /api/v1/documents/tasks/{task_id} 查询。
|
||
"""
|
||
original_filename = file.filename or "unnamed"
|
||
ext = Path(original_filename).suffix.lower()
|
||
|
||
# 1. 扩展名校验
|
||
allowed = _allowed_extensions()
|
||
if ext not in allowed:
|
||
raise ApiError(1001, f"不支持的文件类型: {ext or '(无扩展名)'}")
|
||
|
||
# 2. 读取字节并校验大小
|
||
content = await file.read()
|
||
max_bytes = settings.upload_max_size_mb * 1024 * 1024
|
||
if len(content) > max_bytes:
|
||
raise ApiError(1001, f"文件超过大小上限: {settings.upload_max_size_mb}MB")
|
||
|
||
# 3. 提取文本
|
||
try:
|
||
text = parse_file(original_filename, content)
|
||
except ValueError as exc:
|
||
raise ApiError(1001, str(exc)) from exc
|
||
if not text.strip():
|
||
raise ApiError(1001, "无法从文件提取文本")
|
||
|
||
# 4. 落盘（按 YYYY/MM 日期分片；失败仅 warning，不阻塞入库）
|
||
doc_id = uuid.uuid4().hex
|
||
saved_path = ""
|
||
metadata_dict: dict[str, str] = {}
|
||
try:
|
||
upload_dir = Path(settings.upload_dir).resolve()
|
||
shard_subdir = datetime.now(UTC).strftime("%Y/%m")
|
||
target_dir = upload_dir / shard_subdir
|
||
target_dir.mkdir(parents=True, exist_ok=True)
|
||
target = target_dir / f"{doc_id}_{original_filename}"
|
||
target.write_bytes(content)
|
||
saved_path = str(target)
|
||
metadata_dict.update(
|
||
{
|
||
"raw_file_path": saved_path,
|
||
"original_filename": original_filename,
|
||
"original_size_bytes": str(len(content)),
|
||
}
|
||
)
|
||
logger.info(
|
||
"上传文件已落盘", doc_id=doc_id, saved_path=saved_path, size=len(content)
|
||
)
|
||
except Exception:
|
||
logger.warning(
|
||
"上传文件落盘失败，仅做文本入库",
|
||
doc_id=doc_id,
|
||
filename=original_filename,
|
||
exc_info=True,
|
||
)
|
||
|
||
# 5. 合并用户传入的 metadata（落盘元数据优先级更高，不与用户键冲突）
|
||
if metadata.strip():
|
||
try:
|
||
user_meta = json.loads(metadata)
|
||
if isinstance(user_meta, dict):
|
||
for k, v in user_meta.items():
|
||
metadata_dict.setdefault(str(k), str(v))
|
||
except (json.JSONDecodeError, TypeError):
|
||
logger.warning("metadata 不是合法 JSON，已忽略", raw=metadata)
|
||
|
||
# 6. 默认 title / source
|
||
if not title.strip():
|
||
title = Path(original_filename).stem
|
||
if not source.strip():
|
||
source = f"file:{original_filename}"
|
||
|
||
# 7. 提交入库流水线
|
||
doc_input = DocumentInput(
|
||
text=text, title=title, source=source, metadata=metadata_dict
|
||
)
|
||
task_id = await _get_task_manager().submit(doc_input)
|
||
|
||
return JSONResponse(
|
||
status_code=202,
|
||
content=ok({"task_id": task_id, "status": "pending", "saved_path": saved_path}),
|
||
)
|
||
|
||
|
||
async def _process_single_upload(file: UploadFile) -> dict[str, Any]:
|
||
"""处理单个上传文件：校验 → 提取文本 → 落盘 → 提交入库
|
||
|
||
供批量上传复用：返回成功 {"filename", "task_id"} 或失败 {"filename", "error"}。
|
||
title/source 使用文件名默认值，不接受用户传入的额外表单字段。
|
||
"""
|
||
original_filename = file.filename or "unnamed"
|
||
ext = Path(original_filename).suffix.lower()
|
||
|
||
# 1. 扩展名校验
|
||
allowed = _allowed_extensions()
|
||
if ext not in allowed:
|
||
return {"filename": original_filename, "error": f"不支持的文件类型: {ext or '(无扩展名)'}"}
|
||
|
||
# 2. 读取字节并校验大小
|
||
try:
|
||
content = await file.read()
|
||
except Exception as exc:
|
||
return {"filename": original_filename, "error": f"文件读取失败: {exc}"}
|
||
max_bytes = settings.upload_max_size_mb * 1024 * 1024
|
||
if len(content) > max_bytes:
|
||
return {"filename": original_filename, "error": f"文件超过大小上限: {settings.upload_max_size_mb}MB"}
|
||
|
||
# 3. 提取文本
|
||
try:
|
||
text = parse_file(original_filename, content)
|
||
except ValueError as exc:
|
||
return {"filename": original_filename, "error": str(exc)}
|
||
if not text.strip():
|
||
return {"filename": original_filename, "error": "无法从文件提取文本"}
|
||
|
||
# 4. 落盘（按 YYYY/MM 日期分片；失败仅 warning，不阻塞入库）
|
||
doc_id = uuid.uuid4().hex
|
||
metadata_dict: dict[str, str] = {}
|
||
try:
|
||
upload_dir = Path(settings.upload_dir).resolve()
|
||
shard_subdir = datetime.now(UTC).strftime("%Y/%m")
|
||
target_dir = upload_dir / shard_subdir
|
||
target_dir.mkdir(parents=True, exist_ok=True)
|
||
target = target_dir / f"{doc_id}_{original_filename}"
|
||
target.write_bytes(content)
|
||
metadata_dict.update(
|
||
{
|
||
"raw_file_path": str(target),
|
||
"original_filename": original_filename,
|
||
"original_size_bytes": str(len(content)),
|
||
}
|
||
)
|
||
logger.info(
|
||
"上传文件已落盘", doc_id=doc_id, saved_path=str(target), size=len(content)
|
||
)
|
||
except Exception:
|
||
logger.warning(
|
||
"上传文件落盘失败，仅做文本入库",
|
||
doc_id=doc_id,
|
||
filename=original_filename,
|
||
exc_info=True,
|
||
)
|
||
|
||
# 5. 默认 title / source
|
||
title = Path(original_filename).stem
|
||
source = f"file:{original_filename}"
|
||
|
||
# 6. 提交入库流水线
|
||
doc_input = DocumentInput(text=text, title=title, source=source, metadata=metadata_dict)
|
||
task_id = await _get_task_manager().submit(doc_input)
|
||
return {"filename": original_filename, "task_id": task_id}
|
||
|
||
|
||
@router.post("/documents/upload-batch")
|
||
async def upload_batch(
|
||
files: list[UploadFile] = File(...),
|
||
user: UserRecord = Depends(get_current_user),
|
||
) -> JSONResponse:
|
||
"""批量文件上传入库：逐文件复用单文件逻辑，收集成功与失败结果
|
||
|
||
返回 202 + {tasks: [{filename, task_id}], failed: [{filename, error}]}；
|
||
空文件列表返回 1001。
|
||
"""
|
||
if not files:
|
||
raise ApiError(1001, "未提供任何文件")
|
||
tasks: list[dict[str, Any]] = []
|
||
failed: list[dict[str, Any]] = []
|
||
for file in files:
|
||
result = await _process_single_upload(file)
|
||
if "task_id" in result:
|
||
tasks.append({"filename": result["filename"], "task_id": result["task_id"]})
|
||
else:
|
||
failed.append({"filename": result["filename"], "error": result["error"]})
|
||
return JSONResponse(status_code=202, content=ok({"tasks": tasks, "failed": failed}))
|
||
|
||
|
||
@router.get("/documents/tasks")
|
||
async def list_ingest_tasks(
|
||
limit: int = Query(default=20, ge=1, le=100),
|
||
user: UserRecord = Depends(get_current_user),
|
||
) -> dict[str, Any]:
|
||
"""列出近期入库任务（合并内存与 Redis 镜像，按 updated_at 降序）"""
|
||
tasks = await _get_task_manager().list_tasks(limit=limit)
|
||
return ok({"items": tasks, "total": len(tasks)})
|
||
|
||
|
||
@router.post("/documents/{doc_id}/reingest")
|
||
async def reingest_document(
|
||
doc_id: str, user: UserRecord = Depends(get_current_user)
|
||
) -> JSONResponse:
|
||
"""重新入库：读原始文件 → 删旧数据 → 提交新入库任务
|
||
|
||
仅对有原始文件落盘记录的文档可重新入库；重新入库会生成新 doc_id。
|
||
"""
|
||
meta = await _get_qdrant().get_l1_metadata(doc_id)
|
||
if meta is None:
|
||
raise ApiError(1004, "文档不存在")
|
||
raw_file_path = meta.get("raw_file_path", "")
|
||
if not raw_file_path:
|
||
raise ApiError(1001, "该文档无原始文件，无法重新入库")
|
||
path = Path(raw_file_path)
|
||
if not path.is_file():
|
||
raise ApiError(1004, "文件不存在")
|
||
|
||
# 读原文件并重新提取文本
|
||
try:
|
||
text = parse_file(meta.get("original_filename", path.name), path.read_bytes())
|
||
except ValueError as exc:
|
||
raise ApiError(1001, str(exc)) from exc
|
||
if not text.strip():
|
||
raise ApiError(1001, "无法从文件提取文本")
|
||
|
||
# 删旧数据（四层集合按 doc_id 清除，幂等）
|
||
await _get_qdrant().delete_by_doc_id(doc_id)
|
||
|
||
# 构造新文档输入（保留原 metadata，title/source 从 meta 取或文件名回退）
|
||
original_filename = meta.get("original_filename", path.name)
|
||
title = meta.get("title") or Path(original_filename).stem
|
||
source = meta.get("source") or f"file:{original_filename}"
|
||
doc_input = DocumentInput(text=text, title=title, source=source, metadata=dict(meta))
|
||
|
||
task_id = await _get_task_manager().submit(doc_input)
|
||
return JSONResponse(
|
||
status_code=202, content=ok({"task_id": task_id, "status": "pending"})
|
||
)
|
||
|
||
|
||
@router.get("/documents/tasks/{task_id}")
|
||
async def get_ingest_task(task_id: str) -> dict[str, Any]:
|
||
"""查询入库任务状态：含 task_id/status/created_at/updated_at，done 附 result，failed 附 error"""
|
||
task = await _get_task_manager().get(task_id)
|
||
if task is None:
|
||
raise ApiError(1004, "任务不存在")
|
||
return ok(task)
|
||
|
||
|
||
@router.get("/documents")
|
||
async def list_documents(
|
||
limit: int = Query(default=20, ge=1, le=100),
|
||
offset: str | None = None,
|
||
) -> dict[str, Any]:
|
||
"""分页列出文档（L1 摘要），返回 items 与下一页游标 next_offset"""
|
||
try:
|
||
items, next_offset = await _get_qdrant().scroll_l1(limit=limit, offset=offset)
|
||
except Exception as exc:
|
||
logger.error("文档列表查询失败", error=str(exc))
|
||
raise ApiError(2000, f"文档列表查询失败: {exc}") from exc
|
||
return ok({"items": items, "next_offset": next_offset})
|
||
|
||
|
||
@router.get("/documents/{doc_id}")
|
||
async def get_document(doc_id: str) -> dict[str, Any]:
|
||
"""获取文档详情：L1 记录 + L2/L3 节点 + chunks 数量"""
|
||
try:
|
||
detail = await _get_qdrant().get_doc_detail(doc_id)
|
||
except Exception as exc:
|
||
logger.error("文档详情查询失败", doc_id=doc_id, error=str(exc))
|
||
raise ApiError(2000, f"文档详情查询失败: {exc}") from exc
|
||
if detail is None:
|
||
raise ApiError(1004, "文档不存在")
|
||
return ok(detail)
|
||
|
||
|
||
@router.get("/documents/{doc_id}/file")
|
||
async def download_document_file(doc_id: str) -> FileResponse:
|
||
"""下载文档关联的原始文件（免登录）
|
||
|
||
从 L1 metadata 读取 raw_file_path，校验路径位于 upload_dir 之内后返回 FileResponse；
|
||
文档不存在 / 无关联文件 / 文件缺失 / 路径越界统一返回 1004。
|
||
"""
|
||
meta = await _get_qdrant().get_l1_metadata(doc_id)
|
||
if not meta:
|
||
raise ApiError(1004, "文档不存在或未关联文件")
|
||
raw_path = meta.get("raw_file_path", "")
|
||
if not raw_path:
|
||
raise ApiError(1004, "文档未关联文件")
|
||
path = Path(raw_path).resolve()
|
||
# 路径越界校验：只允许读取 upload_dir 下的文件
|
||
try:
|
||
upload_dir = Path(settings.upload_dir).resolve()
|
||
path.relative_to(upload_dir)
|
||
except ValueError:
|
||
logger.warning("文件路径越界", doc_id=doc_id, raw_path=raw_path)
|
||
raise ApiError(1004, "文件不存在") from None
|
||
if not path.is_file():
|
||
raise ApiError(1004, "文件不存在")
|
||
filename = meta.get("original_filename", path.name)
|
||
return FileResponse(path, filename=filename)
|
||
|
||
|
||
@router.delete("/documents/{doc_id}")
|
||
async def delete_document(
|
||
doc_id: str, user: UserRecord = Depends(get_current_user)
|
||
) -> dict[str, Any]:
|
||
"""删除文档：四层集合中该 doc_id 的所有点；幂等，不存在也返回成功（删除数全 0）"""
|
||
try:
|
||
deleted = await _get_qdrant().delete_by_doc_id(doc_id)
|
||
except Exception as exc:
|
||
logger.error("文档删除失败", doc_id=doc_id, error=str(exc))
|
||
raise ApiError(2000, f"文档删除失败: {exc}") from exc
|
||
return ok(
|
||
{"doc_id": doc_id, "deleted": deleted, "deleted_total": sum(deleted.values())}
|
||
)
|