feat: 新增多格式文件上传入库与认证体系

- 新增 JWT 认证模块，支持登录/注册/用户管理
- 新增文件上传接口，支持 .txt/.md/.html/.pdf/.docx 等格式解析入库
- 新增检索结果 AI 总结功能
- 新增文本去重缓存机制
- 新增全局认证夹具简化测试
- 新增配置项与环境变量支持
- 完善文档与测试覆盖

tests/test_ingest_tasks.py

@@ -1,12 +1,13 @@
 """IngestTaskManager 单元测试（FakeIngester/FakeRedis，不真实联网）"""
 
 import asyncio
+import hashlib
 from collections.abc import Callable
 from datetime import datetime
 from typing import Any
 
 from app.config import Settings
-from app.core.ingest_tasks import REDIS_KEY_PREFIX, IngestTaskManager, IngestTaskStatus
+from app.core.ingest_tasks import DEDUP_KEY_PREFIX, REDIS_KEY_PREFIX, IngestTaskManager, IngestTaskStatus
 from app.core.ingestion import IngestionError
 from app.models.document import DocumentInput, DocumentSummary, IngestionResult, SummaryLevel
 
@@ -198,3 +199,91 @@ async def test_get_falls_back_to_redis_then_none() -> None:
     record = await manager.get("abc")
     assert record is not None
     assert record["status"] == "done"
+
+
+def _text_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+async def test_dedup_hit_reuses_old_doc_id_and_skips_pipeline() -> None:
+    """去重命中：相同 text 第二次提交直接 done，复用旧 doc_id，deduplicated=True，不调 ingester"""
+    ingester = FakeIngester()
+    redis = FakeRedis()
+    manager = IngestTaskManager(ingester, redis, Settings())
+
+    # 首次提交：正常跑流水线
+    text = "重复内容"
+    task1 = await manager.submit(DocumentInput(text=text, title="t1"))
+    final1 = await manager.wait_done(task1, timeout=5)
+    assert final1["status"] == IngestTaskStatus.DONE
+    assert final1["result"]["document_id"] == "doc-1"
+    assert final1["result"]["deduplicated"] is False
+    assert len(ingester.calls) == 1
+
+    # dedup 记录已写入 Redis
+    stored = await redis.get_json(f"{DEDUP_KEY_PREFIX}{_text_hash(text)}")
+    assert stored is not None
+    assert stored["document_id"] == "doc-1"
+
+    # 第二次提交相同 text：直接 done，不重跑流水线
+    task2 = await manager.submit(DocumentInput(text=text, title="t2"))
+    # wait_done 会先 await 所有 fire-and-forget 镜像任务再返回，确保 Redis 已写
+    record2 = await manager.wait_done(task2, timeout=5)
+    assert record2["status"] == IngestTaskStatus.DONE
+    assert record2["result"]["document_id"] == "doc-1"
+    assert record2["result"]["deduplicated"] is True
+    # ingester 没被再次调用
+    assert len(ingester.calls) == 1
+    # 镜像已写入 Redis
+    mirrored = await redis.get_json(f"{REDIS_KEY_PREFIX}{task2}")
+    assert mirrored is not None
+    assert mirrored["status"] == IngestTaskStatus.DONE
+    assert mirrored["result"]["deduplicated"] is True
+
+
+async def test_dedup_miss_when_text_differs() -> None:
+    """去重未命中：不同 text 走原 _run，完成后写入对应 dedup key"""
+    ingester = FakeIngester()
+    redis = FakeRedis()
+    manager = IngestTaskManager(ingester, redis, Settings())
+
+    task1 = await manager.submit(DocumentInput(text="内容A", title="t1"))
+    await manager.wait_done(task1, timeout=5)
+    task2 = await manager.submit(DocumentInput(text="内容B", title="t2"))
+    await manager.wait_done(task2, timeout=5)
+
+    assert await redis.get_json(f"{DEDUP_KEY_PREFIX}{_text_hash('内容A')}") is not None
+    assert await redis.get_json(f"{DEDUP_KEY_PREFIX}{_text_hash('内容B')}") is not None
+    assert len(ingester.calls) == 2
+
+
+async def test_dedup_skipped_when_redis_unavailable() -> None:
+    """Redis 不可用：跳过去重，相同 text 仍走完整流水线"""
+    ingester = FakeIngester()
+    manager = IngestTaskManager(ingester, None, Settings())
+
+    task1 = await manager.submit(DocumentInput(text="内容", title="t1"))
+    final1 = await manager.wait_done(task1, timeout=5)
+    assert final1["result"]["deduplicated"] is False
+
+    task2 = await manager.submit(DocumentInput(text="内容", title="t2"))
+    final2 = await manager.wait_done(task2, timeout=5)
+    assert final2["status"] == IngestTaskStatus.DONE
+    assert final2["result"]["deduplicated"] is False
+    assert len(ingester.calls) == 2
+
+
+async def test_dedup_lookup_failure_falls_back_to_normal_pipeline() -> None:
+    """Redis get_json 抛错：去重查询降级为未命中，走原流水线"""
+    class _ExplodingRedis(FakeRedis):
+        async def get_json(self, key: str) -> dict[str, Any] | None:
+            raise RuntimeError("redis down")
+
+    ingester = FakeIngester()
+    redis = _ExplodingRedis()
+    manager = IngestTaskManager(ingester, redis, Settings())
+
+    task = await manager.submit(DocumentInput(text="x", title="t"))
+    final = await manager.wait_done(task, timeout=5)
+    assert final["status"] == IngestTaskStatus.DONE
+    assert len(ingester.calls) == 1