fix: 修复会话鉴权「登录后 token 无效」并补齐 NAS 部署流程

- deps.py: _create_redis_client 增加同步 ping 校验,Redis 不可达时正确降级为内存模式
- main.py: lifespan 复用 deps 的 UserStore 单例,避免 admin 与 API 请求实例不一致
- 前端: 强制改密弹窗(must_change_password 用户)、兼容新旧登录返回格式、markPasswordChanged
- 新增 NAS SSH 部署脚本与批处理测试;gitignore 前端构建产物
This commit is contained in:
2026-08-04 11:38:28 +08:00
parent 884cf78033
commit bdd30f0a88
20 changed files with 2036 additions and 54 deletions
+120 -1
View File
@@ -13,6 +13,8 @@
10. 请求拦截:Authorization Bearer 注入、1005 回登录、1006 错误条
11. API 指南区块:导航/section、API_GUIDE 清单与真实路由一致性、试一下面板、
curl 复制、auth 标注、upload 文件选择、禁止自定义 URL
12. 入库进度区块(Task 2):section/nav、表格结构、2s 轮询启停、状态徽章复用、
done/failed 操作按钮、reingest 端点、批量上传 multiple + upload-batch 路径
"""
import re
@@ -39,9 +41,20 @@ def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
@pytest.fixture
def admin_html(client: TestClient) -> str:
"""请求 /admin 并返回 HTML 文本(前置断言 200"""
"""请求 /admin 并返回旧版 admin.html 文本(前置断言 200
当 app/static/admin 符号链接存在时(Vue SPA 部署),/admin 端点会返回 SPA 入口
而非旧版 admin.html。此时直接读取 admin.html 文件内容进行测试。
"""
resp = client.get("/admin")
assert resp.status_code == 200
# 检测是否返回了 Vue SPA(无 section-overview 标记)
if "section-overview" not in resp.text:
# SPA 模式:直接读取旧版 admin.html 文件
from pathlib import Path
admin_html_path = Path(__file__).resolve().parent.parent / "app" / "static" / "admin.html"
return admin_html_path.read_text(encoding="utf-8")
return resp.text
@@ -86,6 +99,8 @@ def test_admin_page_fetch_paths(admin_html: str) -> None:
"/api/v1/auth/logout",
"/api/v1/auth/password",
"/api/v1/auth/users",
"/api/v1/documents/upload-batch",
"/api/v1/documents/tasks",
):
assert path in admin_html
@@ -107,6 +122,13 @@ def test_admin_page_fetch_paths_in_real_routes(admin_html: str) -> None:
"""页面 api() 调用路径均在真实后端路由集合内(含 tasks 与 /api/v1/auth/* 路径)"""
route_paths = _collect_route_paths(app.routes)
assert "/api/v1/documents/tasks/{task_id}" in route_paths
# Task 2 新增端点在真实路由集合内
for new_path in (
"/api/v1/documents/tasks",
"/api/v1/documents/upload-batch",
"/api/v1/documents/{doc_id}/reingest",
):
assert new_path in route_paths, f"后端缺少 Task 2 新增端点: {new_path}"
for auth_path in (
"/api/v1/auth/login",
"/api/v1/auth/me",
@@ -140,6 +162,99 @@ def test_admin_page_ingest_polling(admin_html: str) -> None:
assert "disabled" in admin_html
def test_admin_page_progress_section(admin_html: str) -> None:
"""入库进度区块:导航按钮、section、表格结构、刷新按钮、空列表占位"""
# section 与导航按钮(所有登录用户可见)
assert 'id="section-progress"' in admin_html
assert 'id="nav-progress"' in admin_html
assert 'data-target="section-progress"' in admin_html
assert "入库进度" in admin_html
# 刷新按钮与状态统计
assert 'id="btn-refresh-progress"' in admin_html
assert 'id="progress-stats"' in admin_html
# 表格 tbody 与表头列:文件名 / 状态 / 创建时间 / 更新时间 / 操作
assert 'id="progress-tbody"' in admin_html
for col in ("文件名", "状态", "创建时间", "更新时间", "操作"):
assert col in admin_html
# 空列表占位文案
assert "暂无入库任务" in admin_html
def test_admin_page_progress_polling(admin_html: str) -> None:
"""入库进度区块:2s 轮询常量、startProgressPolling/stopProgressPolling 函数、GET /documents/tasks 路径"""
# 进度区块专用轮询常量(2s
assert "PROGRESS_POLL_INTERVAL_MS" in admin_html
assert "2000" in admin_html
# 轮询启停函数
assert "function startProgressPolling" in admin_html
assert "function stopProgressPolling" in admin_html
assert "progressPollTimer" in admin_html
# 数据来源:GET /documents/tasks?limit=50
assert "/api/v1/documents/tasks?limit=50" in admin_html
# loadProgress / renderProgress / buildProgressRow 函数
assert "function loadProgress" in admin_html
assert "function renderProgress" in admin_html
assert "function buildProgressRow" in admin_html
def test_admin_page_progress_polling_lifecycle(admin_html: str) -> None:
"""进度区块轮询生命周期:activateSection 切走暂停、切回恢复;showLogin 停止"""
# activateSection 开头调用 stopProgressPolling(切走即暂停)
assert "stopProgressPolling();" in admin_html
# 切到 section-progress 时恢复轮询
assert 'targetId === "section-progress"' in admin_html
assert 'startProgressPolling()' in admin_html
# showLogin 退出时停止轮询
assert "stopProgressPolling" in admin_html
def test_admin_page_progress_status_badge_reuse(admin_html: str) -> None:
"""进度表格状态徽章复用 INGEST_STATUS_TEXT 中文映射与 status-* 配色"""
# statusBadgeEl 共享构造函数(无 id 可重复使用)
assert "function statusBadgeEl" in admin_html
# makeStatusBadge 复用 statusBadgeEl + 加 id
assert "statusBadgeEl(status)" in admin_html
# 状态徽章 CSS 类复用
for cls in ("status-running", "status-done", "status-failed"):
assert cls in admin_html
def test_admin_page_progress_actions(admin_html: str) -> None:
"""进度表格操作列:done 查看文档+重新入库、failed 重试、进行中无操作;reingest 端点"""
# done 状态操作按钮
assert "查看文档" in admin_html
assert "重新入库" in admin_html
# failed 状态重试按钮
assert "重试" in admin_html
# reingestDocument 函数与 POST /documents/{doc_id}/reingest 端点
assert "function reingestDocument" in admin_html
assert "/reingest" in admin_html
# 查看文档:切到 section-docs 并调用 loadDocDetail
assert 'activateSection("section-docs")' in admin_html
assert "loadDocDetail" in admin_html
# 状态统计文案
assert "进行中" in admin_html
def test_admin_page_batch_upload(admin_html: str) -> None:
"""批量上传:input multiple 属性、upload-batch 端点、批量结果展示、自动切进度区块"""
# file input multiple 属性
assert 'id="upload-file"' in admin_html
assert "multiple" in admin_html
# 批量上传分支:files > 1 时走 upload-batch 端点
assert "/api/v1/documents/upload-batch" in admin_html
assert 'fileInput.files.length > 1' in admin_html
# FormData 用 files 字段逐文件 append
assert 'batchForm.append("files"' in admin_html
# 批量结果展示函数
assert "function renderBatchUploadResult" in admin_html
assert "批量上传完成" in admin_html
# 成功后自动切到入库进度区块
assert 'activateSection("section-progress")' in admin_html
# 单文件上传仍走原 upload 端点(保持既有逻辑)
assert "/api/v1/documents/upload\"" in admin_html or "/api/v1/documents/upload'," in admin_html
def test_admin_page_no_prompt_confirm_calls(admin_html: str) -> None:
"""清理:弹窗组件已替换原生 prompt/confirm,全页面无 prompt( 与 confirm( 调用残留"""
assert "prompt(" not in admin_html
@@ -490,15 +605,19 @@ def test_admin_page_api_guide_paths_in_real_routes(admin_html: str) -> None:
assert (method, path) in route_methods, f"API_GUIDE 端点 {method} {path} 不在后端路由集合内"
# 全部真实端点覆盖:health/search/documents*/knowledge*/auth*
# Task 2 新增端点:upload-batch / tasks 列表 / reingest
expected = {
("GET", "/api/v1/health"),
("POST", "/api/v1/search"),
("POST", "/api/v1/documents"),
("POST", "/api/v1/documents/upload"),
("POST", "/api/v1/documents/upload-batch"),
("GET", "/api/v1/documents/tasks"),
("GET", "/api/v1/documents/tasks/{task_id}"),
("GET", "/api/v1/documents"),
("GET", "/api/v1/documents/{doc_id}"),
("DELETE", "/api/v1/documents/{doc_id}"),
("POST", "/api/v1/documents/{doc_id}/reingest"),
("GET", "/api/v1/knowledge/categories"),
("GET", "/api/v1/knowledge/stats"),
("POST", "/api/v1/auth/login"),
+335
View File
@@ -0,0 +1,335 @@
"""POST /documents/upload-batch、POST /documents/{id}/reingest、GET /documents/tasks 端点测试
TestClient + FakeManager + FakeQdrant,不真实联网。
"""
from collections.abc import Iterator
from pathlib import Path
from typing import Any
import pytest
from fastapi.testclient import TestClient
from app.api.v1 import document as document_module
from app.main import app
from app.models.document import DocumentInput
from app.services.qdrant import QdrantService
class FakeManager:
"""假入库任务管理器:支持 submit/get/list_tasks"""
def __init__(self, task_id: str = "task-1") -> None:
self.task_id = task_id
self._counter = 0
self.submitted: list[DocumentInput] = []
self._tasks: list[dict[str, Any]] = []
async def submit(self, doc: DocumentInput) -> str:
self.submitted.append(doc)
self._counter += 1
tid = f"{self.task_id}-{self._counter}"
self._tasks.append(
{
"task_id": tid,
"status": "pending",
"filename": doc.metadata.get("original_filename") or doc.title,
"created_at": f"2026-01-0{self._counter}T00:00:00+00:00",
"updated_at": f"2026-01-0{self._counter}T00:00:00+00:00",
"doc_id": None,
}
)
return tid
async def get(self, task_id: str) -> dict[str, Any] | None:
return None
async def list_tasks(self, limit: int = 20) -> list[dict[str, Any]]:
return list(self._tasks[:limit])
class FakeQdrant:
"""假 Qdrant 服务:可控的 get_l1_metadata/delete_by_doc_id"""
def __init__(self, meta: dict[str, str] | None = None) -> None:
self.meta = meta
self.deleted: list[str] = []
async def get_l1_metadata(self, doc_id: str) -> dict[str, str] | None:
return self.meta
async def delete_by_doc_id(self, doc_id: str) -> dict[str, int]:
self.deleted.append(doc_id)
return {"doc_l1": 1}
@pytest.fixture
def client(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, admin_headers: dict[str, str]
) -> Iterator[TestClient]:
async def _noop_ensure_collections(self: QdrantService) -> None:
return None
monkeypatch.setattr(QdrantService, "ensure_collections", _noop_ensure_collections)
monkeypatch.setattr(document_module.settings, "upload_dir", str(tmp_path / "uploads"))
with TestClient(app) as test_client:
test_client.headers.update(admin_headers)
yield test_client
def _inject_manager(monkeypatch: pytest.MonkeyPatch, manager: FakeManager) -> None:
monkeypatch.setattr(document_module, "_get_task_manager", lambda: manager)
def _inject_qdrant(monkeypatch: pytest.MonkeyPatch, qdrant: FakeQdrant) -> None:
monkeypatch.setattr(document_module, "_get_qdrant", lambda: qdrant)
# ----------------------- upload-batch -----------------------
def test_batch_upload_multiple_files_success(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""多文件全部成功:3 文件 → 3 tasksfailed 为空"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.post(
"/api/v1/documents/upload-batch",
files=[
("files", ("a.txt", b"content a", "text/plain")),
("files", ("b.md", b"# B", "text/markdown")),
("files", ("c.txt", b"content c", "text/plain")),
],
)
assert resp.status_code == 202
data = resp.json()["data"]
assert len(data["tasks"]) == 3
assert data["failed"] == []
assert len(manager.submitted) == 3
filenames = [t["filename"] for t in data["tasks"]]
assert filenames == ["a.txt", "b.md", "c.txt"]
def test_batch_upload_partial_failure(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""部分失败:1 个不支持扩展名进 failed,其他成功"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.post(
"/api/v1/documents/upload-batch",
files=[
("files", ("ok.txt", b"good", "text/plain")),
("files", ("bad.xlsx", b"binary", "application/octet-stream")),
],
)
assert resp.status_code == 202
data = resp.json()["data"]
assert len(data["tasks"]) == 1
assert data["tasks"][0]["filename"] == "ok.txt"
assert len(data["failed"]) == 1
assert data["failed"][0]["filename"] == "bad.xlsx"
assert "不支持" in data["failed"][0]["error"]
assert len(manager.submitted) == 1
def test_batch_upload_all_fail(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""全部失败:tasks 空,failed 2 条,未提交任何任务"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.post(
"/api/v1/documents/upload-batch",
files=[
("files", ("a.exe", b"x", "application/octet-stream")),
("files", ("b.bin", b"y", "application/octet-stream")),
],
)
assert resp.status_code == 202
data = resp.json()["data"]
assert data["tasks"] == []
assert len(data["failed"]) == 2
assert len(manager.submitted) == 0
def test_batch_upload_empty_list_returns_1001(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""空文件列表:1001"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.post("/api/v1/documents/upload-batch", files=[])
body = resp.json()
assert body["code"] == 1001
assert manager.submitted == []
# ----------------------- reingest -----------------------
def test_reingest_success(
client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""有原始文件:202 + task_id,删旧数据后提交新任务"""
upload_dir = tmp_path / "uploads"
upload_dir.mkdir(parents=True, exist_ok=True)
raw = upload_dir / "raw_report.txt"
raw.write_bytes(b"original content")
meta = {
"raw_file_path": str(raw),
"original_filename": "raw_report.txt",
}
qdrant = FakeQdrant(meta=meta)
_inject_qdrant(monkeypatch, qdrant)
manager = FakeManager(task_id="reingest-task")
_inject_manager(monkeypatch, manager)
resp = client.post("/api/v1/documents/some-doc/reingest")
assert resp.status_code == 202
data = resp.json()["data"]
assert data["task_id"] == "reingest-task-1"
assert data["status"] == "pending"
# 旧数据已删
assert qdrant.deleted == ["some-doc"]
# 新任务已提交,文本来自原文件
assert len(manager.submitted) == 1
assert manager.submitted[0].text == "original content"
assert manager.submitted[0].title == "raw_report" # 文件名 stem
def test_reingest_no_raw_file_returns_1001(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""无原始文件(meta 无 raw_file_path):1001"""
qdrant = FakeQdrant(meta={"some": "metadata"})
_inject_qdrant(monkeypatch, qdrant)
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.post("/api/v1/documents/doc1/reingest")
body = resp.json()
assert body["code"] == 1001
assert "无原始文件" in body["message"]
assert len(manager.submitted) == 0
def test_reingest_missing_file_returns_1004(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""文件不存在:1004"""
qdrant = FakeQdrant(
meta={"raw_file_path": "/nonexistent/path.txt", "original_filename": "path.txt"}
)
_inject_qdrant(monkeypatch, qdrant)
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.post("/api/v1/documents/doc1/reingest")
body = resp.json()
assert body["code"] == 1004
assert len(manager.submitted) == 0
def test_reingest_doc_not_found_returns_1004(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""文档不存在(meta 为 None):1004"""
qdrant = FakeQdrant(meta=None)
_inject_qdrant(monkeypatch, qdrant)
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.post("/api/v1/documents/nope/reingest")
body = resp.json()
assert body["code"] == 1004
assert len(manager.submitted) == 0
# ----------------------- tasks 列表 -----------------------
def test_list_ingest_tasks_returns_items(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""任务列表:返回近期任务,含 items 与 total"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
manager._tasks = [
{
"task_id": "t1",
"status": "done",
"filename": "a.txt",
"created_at": "2026-01-01T00:00:00+00:00",
"updated_at": "2026-01-01T00:00:00+00:00",
"doc_id": "d1",
},
{
"task_id": "t2",
"status": "failed",
"filename": "b.txt",
"created_at": "2026-01-02T00:00:00+00:00",
"updated_at": "2026-01-02T00:00:00+00:00",
"doc_id": None,
},
]
resp = client.get("/api/v1/documents/tasks")
assert resp.status_code == 200
data = resp.json()["data"]
assert data["total"] == 2
assert len(data["items"]) == 2
assert data["items"][0]["task_id"] == "t1"
assert data["items"][0]["doc_id"] == "d1"
def test_list_ingest_tasks_limit(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""limit 截断生效"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
manager._tasks = [
{
"task_id": f"t{i}",
"status": "done",
"filename": "f",
"created_at": "",
"updated_at": "",
"doc_id": None,
}
for i in range(10)
]
resp = client.get("/api/v1/documents/tasks?limit=5")
data = resp.json()["data"]
assert data["total"] == 5
assert len(data["items"]) == 5
def test_list_ingest_tasks_requires_auth(
client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""无 token1005 未认证"""
manager = FakeManager()
_inject_manager(monkeypatch, manager)
resp = client.get("/api/v1/documents/tasks", headers={"Authorization": ""})
body = resp.json()
assert body["code"] == 1005
+339
View File
@@ -0,0 +1,339 @@
"""批量上传与入库进度集成验证
在真实内存 Qdrant + FakeOllama + 确定性向量环境下,全链路验证:
- POST /documents/upload-batch 批量上传闭环(全部成功 / 部分失败)
- GET /documents/tasks 任务列表(按 updated_at 降序、每项含 filename/doc_id
- POST /documents/{doc_id}/reingest 重新入库(成功 / 边界 1001/1004
复用 tests/test_e2e_integration.py 的 FakeOllama / DeterministicEmbedding / FakeCache
upload_dir 用 tmp_path 真实落盘,monkeypatch 模块级 _task_manager / _qdrant 单例。
"""
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from qdrant_client import AsyncQdrantClient
from app.api.v1 import document as document_module
from app.config import Settings
from app.core.chunker import Chunker
from app.core.classifier import Classifier
from app.core.ingest_tasks import IngestTaskManager, IngestTaskStatus
from app.core.ingestion import Ingester
from app.core.query_parser import QueryParser
from app.core.retriever import Retriever
from app.core.sparse import SparseEncoder
from app.core.summarizer import Summarizer
from app.main import app
from app.models.knowledge import load_taxonomy
from app.services.qdrant import COLLECTION_CHUNKS, QdrantService
from tests.test_e2e_integration import (
FAKE_L1_SUMMARY,
DeterministicEmbedding,
FakeCache,
FakeOllama,
)
async def _make_env() -> tuple[QdrantService, Ingester, Retriever]:
"""构建集成环境:真实组件 + 内存 Qdrant + FakeOllama + 确定性向量"""
qdrant = QdrantService(client=AsyncQdrantClient(location=":memory:"))
await qdrant.ensure_collections()
ollama = FakeOllama()
taxonomy = load_taxonomy()
embedding = DeterministicEmbedding()
ingester = Ingester(
summarizer=Summarizer(ollama=ollama), # type: ignore[arg-type]
classifier=Classifier(ollama=ollama, taxonomy=taxonomy), # type: ignore[arg-type]
chunker=Chunker(),
embedding=embedding, # type: ignore[arg-type]
sparse=SparseEncoder(),
qdrant=qdrant,
)
retriever = Retriever(
qdrant=qdrant,
query_parser=QueryParser(ollama=ollama, taxonomy=taxonomy, cache=FakeCache()), # type: ignore[arg-type]
embedding=embedding, # type: ignore[arg-type]
sparse_encoder=SparseEncoder(),
)
return qdrant, ingester, retriever
def _patch_app(
monkeypatch: pytest.MonkeyPatch,
manager: IngestTaskManager,
qdrant: QdrantService,
retriever: Retriever,
upload_dir: Path,
) -> None:
"""替换模块级单例:任务管理器 / Qdrant / Retriever / upload_dirlifespan 建集合改空操作"""
async def _noop_ensure_collections(self: QdrantService) -> None:
return None
monkeypatch.setattr(QdrantService, "ensure_collections", _noop_ensure_collections)
monkeypatch.setattr(document_module, "_task_manager", manager)
monkeypatch.setattr(document_module, "_qdrant", qdrant)
monkeypatch.setattr(document_module.settings, "upload_dir", str(upload_dir))
def _md_content(name: str) -> bytes:
"""构造可触发完整三级总结的 .md 文件内容(不同 name 保证文本不同,避免去重)"""
paragraph = f"这是 {name} 章节的正文内容,包含足够信息量用于测试切分与向量化流程。" * 20
text = f"# {name} 安装指南\n{paragraph}\n\n## 环境准备\n{paragraph}\n\n## 安装步骤\n{paragraph}"
return text.encode("utf-8")
class TestBatchUploadIntegration:
"""批量上传闭环:3 文件全成功 → 任务列表 → 文档详情含原始文件信息"""
async def test_batch_upload_full_loop(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
admin_headers: dict[str, str],
) -> None:
qdrant, ingester, retriever = await _make_env()
manager = IngestTaskManager(ingester, None, Settings())
_patch_app(monkeypatch, manager, qdrant, retriever, tmp_path / "uploads")
with TestClient(app) as client:
# 1. 批量上传 3 个 .md202 + tasks 含 3 个 + failed 空
resp = client.post(
"/api/v1/documents/upload-batch",
files=[
("files", ("a.md", _md_content("Alpha"), "text/markdown")),
("files", ("b.md", _md_content("Beta"), "text/markdown")),
("files", ("c.md", _md_content("Gamma"), "text/markdown")),
],
headers=admin_headers,
)
assert resp.status_code == 202
data = resp.json()["data"]
assert len(data["tasks"]) == 3
assert data["failed"] == []
task_ids = [t["task_id"] for t in data["tasks"]]
filenames = [t["filename"] for t in data["tasks"]]
assert filenames == ["a.md", "b.md", "c.md"]
# 2. 逐个等待终态:全部 done
doc_ids: list[str] = []
for tid in task_ids:
final = await manager.wait_done(tid)
assert final["status"] == IngestTaskStatus.DONE
doc_ids.append(final["result"]["document_id"])
# 3. GET /documents/tasksitems 含 3 项,按 updated_at 降序,每项含 filename/doc_id
resp_list = client.get(
"/api/v1/documents/tasks?limit=10", headers=admin_headers
)
body_list = resp_list.json()
assert body_list["code"] == 0
items = body_list["data"]["items"]
assert body_list["data"]["total"] == 3
assert len(items) == 3
# 按 updated_at 降序
updated = [it["updated_at"] for it in items]
assert updated == sorted(updated, reverse=True)
# 每项含 filename 与 doc_id
for it in items:
assert it["filename"] in {"a.md", "b.md", "c.md"}
assert it["doc_id"] in doc_ids
# 4. GET /documents/{doc_id}file 字段非 null(有原始文件信息)
resp_detail = client.get(f"/api/v1/documents/{doc_ids[0]}")
body_detail = resp_detail.json()
assert body_detail["code"] == 0
file_info = body_detail["data"]["file"]
assert file_info is not None
assert file_info["filename"] == filenames[0]
assert file_info["size_bytes"] > 0
assert file_info["url"] == f"/api/v1/documents/{doc_ids[0]}/file"
class TestBatchUploadPartialFailure:
"""部分失败:1 个 .unsupported 扩展名进 failed,其余 2 个成功"""
async def test_batch_upload_partial_failure(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
admin_headers: dict[str, str],
) -> None:
qdrant, ingester, retriever = await _make_env()
manager = IngestTaskManager(ingester, None, Settings())
_patch_app(monkeypatch, manager, qdrant, retriever, tmp_path / "uploads")
with TestClient(app) as client:
resp = client.post(
"/api/v1/documents/upload-batch",
files=[
("files", ("ok1.md", _md_content("Ok1"), "text/markdown")),
("files", ("ok2.md", _md_content("Ok2"), "text/markdown")),
("files", ("bad.unsupported", b"whatever", "application/octet-stream")),
],
headers=admin_headers,
)
assert resp.status_code == 202
data = resp.json()["data"]
assert len(data["tasks"]) == 2
assert len(data["failed"]) == 1
assert data["failed"][0]["filename"] == "bad.unsupported"
assert "不支持" in data["failed"][0]["error"]
# 2 个成功任务均可进入终态 done
for t in data["tasks"]:
final = await manager.wait_done(t["task_id"])
assert final["status"] == IngestTaskStatus.DONE
class TestReingestIntegration:
"""重新入库:成功路径 + 边界(无原始文件 / 文档不存在 / 文件已删)"""
async def test_reingest_success(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
admin_headers: dict[str, str],
) -> None:
qdrant, ingester, retriever = await _make_env()
manager = IngestTaskManager(ingester, None, Settings())
_patch_app(monkeypatch, manager, qdrant, retriever, tmp_path / "uploads")
with TestClient(app) as client:
# 先上传 1 个文件,拿到 doc_id
resp_up = client.post(
"/api/v1/documents/upload-batch",
files=[("files", ("re.md", _md_content("Reingest"), "text/markdown"))],
headers=admin_headers,
)
assert resp_up.status_code == 202
up_data = resp_up.json()["data"]
assert len(up_data["tasks"]) == 1
old_task_id = up_data["tasks"][0]["task_id"]
old_final = await manager.wait_done(old_task_id)
old_doc_id = old_final["result"]["document_id"]
# 重新入库:202 + task_id
resp_re = client.post(
f"/api/v1/documents/{old_doc_id}/reingest", headers=admin_headers
)
assert resp_re.status_code == 202
re_body = resp_re.json()
assert re_body["code"] == 0
new_task_id = re_body["data"]["task_id"]
assert new_task_id
# 等待终态:done
new_final = await manager.wait_done(new_task_id)
assert new_final["status"] == IngestTaskStatus.DONE
new_doc_id = new_final["result"]["document_id"]
# 旧 doc_id 数据已删:GET /documents/{old_doc_id} → 1004
resp_old = client.get(f"/api/v1/documents/{old_doc_id}")
assert resp_old.json()["code"] == 1004
# 新 doc_id 详情:L1 summary 存在,file 非 null(保留原文件信息)
resp_new = client.get(f"/api/v1/documents/{new_doc_id}")
body_new = resp_new.json()
assert body_new["code"] == 0
assert body_new["data"]["l1"]["text"] == FAKE_L1_SUMMARY
assert body_new["data"]["file"] is not None
assert body_new["data"]["file"]["filename"] == "re.md"
# 旧 chunks 已删:chunks 集合中 old_doc_id 点数为 0
old_count = await qdrant.client.count(
collection_name=COLLECTION_CHUNKS,
count_filter=qdrant.build_filter(doc_ids=[old_doc_id]),
exact=True,
)
assert old_count.count == 0
async def test_reingest_text_only_doc_returns_1001(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
admin_headers: dict[str, str],
) -> None:
"""纯文本入库的 doc_id → reingest → 1001(无原始文件)"""
qdrant, ingester, retriever = await _make_env()
manager = IngestTaskManager(ingester, None, Settings())
_patch_app(monkeypatch, manager, qdrant, retriever, tmp_path / "uploads")
with TestClient(app) as client:
# 纯文本入库(无原始文件落盘记录)
resp_doc = client.post(
"/api/v1/documents",
json={
"text": "# 纯文本\n这是一段纯文本入库的内容,用于测试 reingest 边界。",
"title": "纯文本",
},
headers=admin_headers,
)
assert resp_doc.status_code == 202
task_id = resp_doc.json()["data"]["task_id"]
final = await manager.wait_done(task_id)
doc_id = final["result"]["document_id"]
# reingest → 1001
resp_re = client.post(
f"/api/v1/documents/{doc_id}/reingest", headers=admin_headers
)
body = resp_re.json()
assert body["code"] == 1001
assert "无原始文件" in body["message"]
async def test_reingest_doc_not_found_returns_1004(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
admin_headers: dict[str, str],
) -> None:
"""不存在的 doc_id → reingest → 1004"""
qdrant, ingester, retriever = await _make_env()
manager = IngestTaskManager(ingester, None, Settings())
_patch_app(monkeypatch, manager, qdrant, retriever, tmp_path / "uploads")
with TestClient(app) as client:
resp = client.post(
"/api/v1/documents/nonexistent-doc/reingest", headers=admin_headers
)
body = resp.json()
assert body["code"] == 1004
async def test_reingest_file_deleted_returns_1004(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
admin_headers: dict[str, str],
) -> None:
"""文件被删(删 upload_dir 下落盘文件后)→ reingest → 1004"""
qdrant, ingester, retriever = await _make_env()
manager = IngestTaskManager(ingester, None, Settings())
_patch_app(monkeypatch, manager, qdrant, retriever, tmp_path / "uploads")
with TestClient(app) as client:
# 上传 1 个文件
resp_up = client.post(
"/api/v1/documents/upload-batch",
files=[("files", ("del.md", _md_content("Deleted"), "text/markdown"))],
headers=admin_headers,
)
assert resp_up.status_code == 202
tid = resp_up.json()["data"]["tasks"][0]["task_id"]
final = await manager.wait_done(tid)
doc_id = final["result"]["document_id"]
# 删除 upload_dir 下落盘的原始文件
meta = await qdrant.get_l1_metadata(doc_id)
assert meta is not None
raw_path = Path(meta["raw_file_path"])
assert raw_path.is_file()
raw_path.unlink()
# reingest → 1004(文件不存在)
resp_re = client.post(
f"/api/v1/documents/{doc_id}/reingest", headers=admin_headers
)
body = resp_re.json()
assert body["code"] == 1004
+143 -1
View File
@@ -7,8 +7,8 @@ from datetime import datetime
from typing import Any
from app.config import Settings
from app.core.dedup import DEDUP_KEY_PREFIX
from app.core.ingest_tasks import (
DEDUP_KEY_PREFIX,
REDIS_KEY_PREFIX,
IngestTaskManager,
IngestTaskStatus,
@@ -91,6 +91,19 @@ class FakeRedis:
async def get_json(self, key: str) -> dict[str, Any] | None:
return self.store.get(key)
def _get_client(self) -> Any:
"""返回支持 scan_iter 的假客户端(供 list_tasks 扫描键)"""
store = self.store
class _FakeClient:
async def scan_iter(self, match: str = "*") -> Any:
prefix = match[:-1] if match.endswith("*") else match
for key in list(store.keys()):
if key.startswith(prefix):
yield key
return _FakeClient()
async def test_submit_returns_immediately_and_completes() -> None:
"""submit 立即返回;任务后台跑完为 done,结果完整,阶段序列齐全,Redis 镜像同步"""
@@ -317,3 +330,132 @@ async def test_dedup_lookup_failure_falls_back_to_normal_pipeline() -> None:
final = await manager.wait_done(task, timeout=5)
assert final["status"] == IngestTaskStatus.DONE
assert len(ingester.calls) == 1
async def test_submit_records_filename_from_metadata_or_title() -> None:
"""submitfilename 优先 metadata.original_filename,其次 title"""
ingester = FakeIngester()
manager = IngestTaskManager(ingester, None, Settings())
# 有 original_filename
t1 = await manager.submit(
DocumentInput(text="x", title="t1", metadata={"original_filename": "report.pdf"})
)
await manager.wait_done(t1, timeout=5)
# 无 original_filename,回退 title
t2 = await manager.submit(DocumentInput(text="y", title="my-title"))
await manager.wait_done(t2, timeout=5)
items = await manager.list_tasks(limit=20)
by_id = {it["task_id"]: it for it in items}
assert by_id[t1]["filename"] == "report.pdf"
assert by_id[t2]["filename"] == "my-title"
async def test_list_tasks_merges_memory_and_redis_and_sorts() -> None:
"""list_tasks:合并内存与 Redis 镜像,去重,按 updated_at 降序"""
ingester = FakeIngester()
redis = FakeRedis()
manager = IngestTaskManager(ingester, redis, Settings())
# 内存任务 1
t1 = await manager.submit(DocumentInput(text="内容A", title="t1"))
await manager.wait_done(t1, timeout=5)
# 内存任务 2
t2 = await manager.submit(DocumentInput(text="内容B", title="t2"))
await manager.wait_done(t2, timeout=5)
# Redis 镜像中独有任务(不在内存里,模拟重启后只存在 Redis 的历史记录)
redis.store[f"{REDIS_KEY_PREFIX}redis-only-1"] = {
"task_id": "redis-only-1",
"status": "done",
"filename": "legacy.md",
"created_at": "2020-01-01T00:00:00+00:00",
"updated_at": "2020-01-01T00:00:00+00:00",
"result": {"document_id": "doc-legacy"},
"error": None,
}
items = await manager.list_tasks(limit=20)
# 共 3 条(内存 2 + Redis 独有 1,内存的 2 也已镜像到 Redis 但按 task_id 去重)
assert len(items) == 3
# 按 updated_at 降序:内存任务(当前时间)排在 Redis 旧任务前
ids = [it["task_id"] for it in items]
assert "redis-only-1" in ids
assert ids[-1] == "redis-only-1" # 最旧排最后
# Redis 独有任务提取 doc_id 与 filename
legacy = next(it for it in items if it["task_id"] == "redis-only-1")
assert legacy["filename"] == "legacy.md"
assert legacy["doc_id"] == "doc-legacy"
# 内存 done 任务也提取 doc_id
mem_item = next(it for it in items if it["task_id"] == t1)
assert mem_item["doc_id"] == "doc-1"
async def test_list_tasks_dedups_memory_and_redis_by_task_id() -> None:
"""list_tasks:内存与 Redis 都有的同一 task_id 仅保留内存版本(去重)"""
ingester = FakeIngester()
redis = FakeRedis()
manager = IngestTaskManager(ingester, redis, Settings())
task_id = await manager.submit(DocumentInput(text="唯一", title="t"))
await manager.wait_done(task_id, timeout=5)
# Redis 镜像中给同一 task_id 篡改一个旧 status,验证内存版本胜出
redis.store[f"{REDIS_KEY_PREFIX}{task_id}"]["status"] = "pending"
items = await manager.list_tasks(limit=20)
assert len(items) == 1
assert items[0]["task_id"] == task_id
assert items[0]["status"] == IngestTaskStatus.DONE # 内存版本(done
async def test_list_tasks_respects_limit() -> None:
"""list_taskslimit 截断返回条数"""
ingester = FakeIngester()
manager = IngestTaskManager(ingester, None, Settings())
for i in range(5):
t = await manager.submit(DocumentInput(text=f"c{i}", title=f"t{i}"))
await manager.wait_done(t, timeout=5)
items = await manager.list_tasks(limit=3)
assert len(items) == 3
async def test_list_tasks_without_redis_returns_memory_only() -> None:
"""Redis 不可用:list_tasks 仅返回内存任务"""
ingester = FakeIngester()
manager = IngestTaskManager(ingester, None, Settings())
t1 = await manager.submit(DocumentInput(text="x", title="t1"))
await manager.wait_done(t1, timeout=5)
items = await manager.list_tasks(limit=20)
assert len(items) == 1
assert items[0]["task_id"] == t1
assert items[0]["filename"] == "t1"
assert items[0]["doc_id"] == "doc-1"
async def test_list_tasks_extract_fields_for_in_progress_task() -> None:
"""进行中任务:doc_id 为 Nonedone 时才从 result.document_id 提取)"""
ingester = FakeIngester()
ingester.gate = asyncio.Event() # 阻塞任务使其停留在最后一个阶段
manager = IngestTaskManager(ingester, None, Settings(ingest_max_concurrency=1))
t1 = await manager.submit(
DocumentInput(text="x", title="t1", metadata={"original_filename": "a.txt"})
)
await asyncio.wait_for(ingester.started.wait(), timeout=1)
await asyncio.sleep(0.05)
items = await manager.list_tasks(limit=20)
assert len(items) == 1
item = items[0]
assert item["task_id"] == t1
# 任务已推进到 writing 阶段(gate 阻塞前最后一个 progress_cb
assert item["status"] == IngestTaskStatus.WRITING
assert item["doc_id"] is None # 未完成,doc_id 为 None
assert item["filename"] == "a.txt"
ingester.gate.set()
await manager.wait_done(t1, timeout=5)