kplam/QMDSearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 统一代码格式，调整多行代码换行风格

Browse Source 
对多个文件进行代码格式化调整，将长行参数拆分为多行书写，提升代码可读性，包括：
- 调整函数定义、调用的多行换行格式
- 优化列表、元组、字典的多行排版
- 新增README.md项目说明文档
This commit is contained in:
kplam
2026-07-30 14:25:12 +08:00
parent dce9e31bde
commit fdb664e546
10 changed files with 491 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/v1/document.py
+44 -13
View File
@@ -57,30 +57,44 @@ def _get_task_manager() -> IngestTaskManager:
except Exception:
logger.warning("Redis 缓存不可用，入库任务状态仅保留在内存", exc_info=True)
redis = None
_task_manager = IngestTaskManager(ingester=_get_ingester(), redis=redis, settings=Settings())
_task_manager = IngestTaskManager(
ingester=_get_ingester(), redis=redis, settings=Settings()
)
return _task_manager
def _allowed_extensions() -> set[str]:
"""解析 settings.upload_allowed_extensions 逗号分隔字符串为扩展名集合（全小写、含点号）"""
return {ext.strip().lower() for ext in settings.upload_allowed_extensions.split(",") if ext.strip()}
return {
ext.strip().lower()
for ext in settings.upload_allowed_extensions.split(",")
if ext.strip()
}
@router.post("/documents")
async def ingest_document(doc: DocumentInput, user: AuthUser = Depends(get_current_user)) -> JSONResponse:
async def ingest_document(
doc: DocumentInput, user: AuthUser = Depends(get_current_user)
) -> JSONResponse:
"""文档入库入口：登记异步任务并返回 202 + task_id，入库结果经任务查询端点获取"""
if not doc.text.strip():
raise ApiError(1001, "文档内容不能为空")
task_id = await _get_task_manager().submit(doc)
return JSONResponse(status_code=202, content=ok({"task_id": task_id, "status": "pending"}))
return JSONResponse(
status_code=202, content=ok({"task_id": task_id, "status": "pending"})
)
@router.post("/documents/upload")
async def upload_document(
file: UploadFile = File(..., description="上传的文件（.txt/.md/.html/.htm/.pdf/.docx）"),
file: UploadFile = File(
..., description="上传的文件（.txt/.md/.html/.htm/.pdf/.docx）"
),
title: str = Form(default="", description="可选标题，默认取原文件名去扩展"),
source: str = Form(default="", description="可选来源标识，默认 file:{原文件名}"),
metadata: str = Form(default="", description='可选元数据 JSON 字符串，如 \'{"author":"x"}\''),
metadata: str = Form(
default="", description='可选元数据 JSON 字符串，如 \'{"author":"x"}\''
),
user: AuthUser = Depends(get_current_user),
) -> JSONResponse:
"""文件上传入库入口：校验 → 提取文本 → 落盘 → 提交异步入库流水线
@@ -129,9 +143,16 @@ async def upload_document(
"original_size_bytes": str(len(content)),
}
)
logger.info("上传文件已落盘", doc_id=doc_id, saved_path=saved_path, size=len(content))
logger.info(
"上传文件已落盘", doc_id=doc_id, saved_path=saved_path, size=len(content)
)
except Exception:
logger.warning("上传文件落盘失败，仅做文本入库", doc_id=doc_id, filename=original_filename, exc_info=True)
logger.warning(
"上传文件落盘失败，仅做文本入库",
doc_id=doc_id,
filename=original_filename,
exc_info=True,
)
# 5. 合并用户传入的 metadata（落盘元数据优先级更高，不与用户键冲突）
if metadata.strip():
@@ -150,7 +171,9 @@ async def upload_document(
source = f"file:{original_filename}"
# 7. 提交入库流水线
doc_input = DocumentInput(text=text, title=title, source=source, metadata=metadata_dict)
doc_input = DocumentInput(
text=text, title=title, source=source, metadata=metadata_dict
)
task_id = await _get_task_manager().submit(doc_input)
return JSONResponse(
@@ -160,7 +183,9 @@ async def upload_document(
@router.get("/documents/tasks/{task_id}")
async def get_ingest_task(task_id: str, user: AuthUser = Depends(get_current_user)) -> dict[str, Any]:
async def get_ingest_task(
task_id: str, user: AuthUser = Depends(get_current_user)
) -> dict[str, Any]:
"""查询入库任务状态：含 task_id/status/created_at/updated_at，done 附 result，failed 附 error"""
task = await _get_task_manager().get(task_id)
if task is None:
@@ -184,7 +209,9 @@ async def list_documents(
@router.get("/documents/{doc_id}")
async def get_document(doc_id: str, user: AuthUser = Depends(get_current_user)) -> dict[str, Any]:
async def get_document(
doc_id: str, user: AuthUser = Depends(get_current_user)
) -> dict[str, Any]:
"""获取文档详情：L1 记录 + L2/L3 节点 + chunks 数量"""
try:
detail = await _get_qdrant().get_doc_detail(doc_id)
@@ -197,11 +224,15 @@ async def get_document(doc_id: str, user: AuthUser = Depends(get_current_user))
@router.delete("/documents/{doc_id}")
async def delete_document(doc_id: str, user: AuthUser = Depends(require_admin)) -> dict[str, Any]:
async def delete_document(
doc_id: str, user: AuthUser = Depends(require_admin)
) -> dict[str, Any]:
"""删除文档：四层集合中该 doc_id 的所有点；幂等，不存在也返回成功（删除数全 0）"""
try:
deleted = await _get_qdrant().delete_by_doc_id(doc_id)
except Exception as exc:
logger.error("文档删除失败", doc_id=doc_id, error=str(exc))
raise ApiError(2000, f"文档删除失败: {exc}") from exc
return ok({"doc_id": doc_id, "deleted": deleted, "deleted_total": sum(deleted.values())})
return ok(
{"doc_id": doc_id, "deleted": deleted, "deleted_total": sum(deleted.values())}
)
app/config.py
+6 -2
View File
@@ -65,10 +65,14 @@ class Settings(BaseSettings):
# 文件上传
upload_dir: str = "./uploads" # 原始文件保存目录（相对路径以工作目录为基）
upload_max_size_mb: int = 20 # 单文件大小上限（MB）
upload_allowed_extensions: str = ".txt,.md,.html,.htm,.pdf,.docx" # 允许上传的扩展名（逗号分隔）
upload_allowed_extensions: str = (
".txt,.md,.html,.htm,.pdf,.docx" # 允许上传的扩展名（逗号分隔）
)
# PDF OCR（图片型/扫描件降级，pypdf extract_text 为空时触发）
pdf_ocr_enabled: bool = True # 是否启用 OCR 降级（关闭则扫描件按"无法提取文本"拒绝入库）
pdf_ocr_enabled: bool = (
True # 是否启用 OCR 降级（关闭则扫描件按"无法提取文本"拒绝入库）
)
pdf_ocr_max_pages: int = 30 # 单文件 OCR 页数上限，超过仅前 N 页
pdf_ocr_dpi: int = 200 # 渲染 DPI（越高越准但越慢，72~300 合理）
app/core/file_parser.py
+6 -2
View File
@@ -102,7 +102,9 @@ def _parse_pdf(content: bytes) -> str:
if not settings.pdf_ocr_enabled:
return ""
ocr_text = _ocr_pdf(content, max_pages=settings.pdf_ocr_max_pages, dpi=settings.pdf_ocr_dpi)
ocr_text = _ocr_pdf(
content, max_pages=settings.pdf_ocr_max_pages, dpi=settings.pdf_ocr_dpi
)
return ocr_text
@@ -143,7 +145,9 @@ def _ocr_pdf(content: bytes, max_pages: int, dpi: int) -> str:
result, _ = _ocr_engine(pil_image)
if result:
# result: [[box, text, score], ...]，按行拼接
lines = [item[1] for item in result if item and len(item) >= 2 and item[1]]
lines = [
item[1] for item in result if item and len(item) >= 2 and item[1]
]
if lines:
page_texts.append("\n".join(lines))
pdf.close()
app/core/ingest_tasks.py
+44 -12
View File
@@ -43,7 +43,9 @@ class IngestTaskStatus(StrEnum):
# 终态集合
TERMINAL_STATUSES: frozenset[str] = frozenset({IngestTaskStatus.DONE, IngestTaskStatus.FAILED})
TERMINAL_STATUSES: frozenset[str] = frozenset(
{IngestTaskStatus.DONE, IngestTaskStatus.FAILED}
)
def _utc_now_iso() -> str:
@@ -54,7 +56,9 @@ def _utc_now_iso() -> str:
class IngestTaskManager:
"""入库异步任务管理器：登记、后台执行、状态查询与 Redis 持久镜像"""
def __init__(self, ingester: Ingester, redis: RedisCache | None, settings: Settings) -> None:
def __init__(
self, ingester: Ingester, redis: RedisCache | None, settings: Settings
) -> None:
self._ingester = ingester
self._redis = redis
self._settings = settings
@@ -123,7 +127,11 @@ class IngestTaskManager:
try:
return await self._redis.get_json(f"{REDIS_KEY_PREFIX}{task_id}")
except Exception:
logger.warning("入库任务状态读取 Redis 失败，降级为未命中", task_id=task_id, exc_info=True)
logger.warning(
"入库任务状态读取 Redis 失败，降级为未命中",
task_id=task_id,
exc_info=True,
)
return None
async def wait_done(self, task_id: str, timeout: float = 30.0) -> dict[str, Any]:
@@ -147,7 +155,9 @@ class IngestTaskManager:
"""后台执行入库：并发限流 + 阶段状态推进 + 结果/错误落账 + 去重记录写入"""
async with self._semaphore:
try:
result = await self._ingester.ingest(doc, progress_cb=lambda stage: self._on_progress(task_id, stage))
result = await self._ingester.ingest(
doc, progress_cb=lambda stage: self._on_progress(task_id, stage)
)
except IngestionError as exc:
# 入库已知失败：透传阶段与已产出的部分总结
self._finish_failed(
@@ -155,11 +165,18 @@ class IngestTaskManager:
{
"stage": exc.stage,
"message": str(exc),
"partial_summary": exc.summary.model_dump(mode="json") if exc.summary is not None else None,
"partial_summary": (
exc.summary.model_dump(mode="json")
if exc.summary is not None
else None
),
},
)
except Exception as exc:
self._finish_failed(task_id, {"stage": "unknown", "message": str(exc), "partial_summary": None})
self._finish_failed(
task_id,
{"stage": "unknown", "message": str(exc), "partial_summary": None},
)
else:
result_dict = result.model_dump(mode="json")
self._tasks[task_id].update(
@@ -178,7 +195,9 @@ class IngestTaskManager:
try:
return await self._redis.get_json(f"{DEDUP_KEY_PREFIX}{text_hash}")
except Exception:
logger.warning("去重记录查询失败，降级为未命中", text_hash=text_hash, exc_info=True)
logger.warning(
"去重记录查询失败，降级为未命中", text_hash=text_hash, exc_info=True
)
return None
async def _record_dedup(self, text_hash: str, result_dict: dict[str, Any]) -> None:
@@ -196,9 +215,16 @@ class IngestTaskManager:
def _finish_failed(self, task_id: str, error: dict[str, Any]) -> None:
"""将任务置为 failed 并记录错误信息"""
self._tasks[task_id].update(status=IngestTaskStatus.FAILED, updated_at=_utc_now_iso(), error=error)
self._tasks[task_id].update(
status=IngestTaskStatus.FAILED, updated_at=_utc_now_iso(), error=error
)
self._schedule_mirror(task_id)
logger.error("入库任务失败", task_id=task_id, stage=error["stage"], error=error["message"])
logger.error(
"入库任务失败",
task_id=task_id,
stage=error["stage"],
error=error["message"],
)
def _on_progress(self, task_id: str, stage: str) -> None:
"""Ingester 阶段回调：推进任务状态并同步镜像（同步函数，供 progress_cb 使用）"""
@@ -213,7 +239,9 @@ class IngestTaskManager:
"""将当前任务状态快照异步镜像到 Redis（同步上下文也可调用）"""
if self._redis is None:
return
mirror = asyncio.create_task(self._mirror_to_redis(task_id, dict(self._tasks[task_id])))
mirror = asyncio.create_task(
self._mirror_to_redis(task_id, dict(self._tasks[task_id]))
)
self._mirror_tasks.add(mirror)
mirror.add_done_callback(self._mirror_tasks.discard)
@@ -225,6 +253,10 @@ class IngestTaskManager:
else self._settings.ingest_task_ttl_done
)
try:
await self._redis.set_json(f"{REDIS_KEY_PREFIX}{task_id}", snapshot, ttl=ttl)
await self._redis.set_json(
f"{REDIS_KEY_PREFIX}{task_id}", snapshot, ttl=ttl
)
except Exception:
logger.warning("入库任务状态镜像 Redis 失败", task_id=task_id, exc_info=True)
logger.warning(
"入库任务状态镜像 Redis 失败", task_id=task_id, exc_info=True
)
app/main.py
+3 -1
View File
@@ -62,7 +62,9 @@ async def api_error_handler(request: Request, exc: ApiError) -> JSONResponse:
@app.exception_handler(RequestValidationError)
async def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
async def validation_error_handler(
request: Request, exc: RequestValidationError
) -> JSONResponse:
"""请求参数校验失败 → code 1001"""
errors = exc.errors()
message = f"请求参数校验失败: {errors[0]['msg']}" if errors else "请求参数校验失败"
app/models/document.py
+7 -2
View File
@@ -16,7 +16,9 @@ class DocumentSummary(BaseModel):
"""文档三级总结结果"""
l1_summary: str = Field(description="L1 总结：一句话高度概括")
l2_outline: str | None = Field(default=None, description="L2 大纲：主要章节和关键主题")
l2_outline: str | None = Field(
default=None, description="L2 大纲：主要章节和关键主题"
)
l3_content_outline: str = Field(description="L3/L2.5 内容大纲：详细内容摘要")
level: SummaryLevel = Field(description="实际使用的总结层级")
@@ -49,4 +51,7 @@ class IngestionResult(BaseModel):
chunks_count: int = Field(default=0, description="写入的 chunk 数量")
tags: list[str] = Field(default_factory=list, description="附加分类标签")
category_confidence: float = Field(default=0.0, description="主类目分类置信度")
deduplicated: bool = Field(default=False, description="是否命中去重复用旧文档（True 时 document_id 为既有文档 ID）")
deduplicated: bool = Field(
default=False,
description="是否命中去重复用旧文档（True 时 document_id 为既有文档 ID）",
)