阶段1:服务化骨架(FastAPI + Playwright worker + 任务队列)
- 新增 runtime.py:抽离共享核心 launch_and_prepare / dispatch_task / run_heartbeat /
probe_* / launch_anneng / RuntimeContext,常量 SITES_CONFIG 等;main_router 复用
- 新增 server.py:FastAPI(主线程)+ Playwright worker(独立线程)+ 任务队列;
API:POST/GET /tasks、GET /status、GET /data/{file}(防路径穿越)
- state_store:加 task_history 表 + create/update/get/list 接口
- 5 站点 with_retry 改为返回 True/False,供 dispatch_task 判成败
- main_router:重写为复用 runtime 的交互模式(行为不变)
- requirements:加 fastapi、uvicorn
- CLAUDE.md:补充运行模式与共享核心架构说明
线程模型:主线程 FastAPI 不碰 Playwright,worker 线程独占 page,经 Queue + SQLite 通信。
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,13 @@ LOGIN_UNKNOWN = "unknown" # 尚未探测过
|
||||
LOGIN_IN = "logged_in"
|
||||
LOGIN_OUT = "logged_out"
|
||||
|
||||
# 任务状态枚举(task_history.status)
|
||||
TASK_PENDING = "pending" # 已入队,待执行
|
||||
TASK_RUNNING = "running" # 正在执行
|
||||
TASK_SUCCESS = "success" # 成功(有数据)
|
||||
TASK_NO_DATA = "no_data" # 成功但本站本次无数据
|
||||
TASK_FAILED = "failed" # 失败(重试耗尽 / 未登录 / 异常)
|
||||
|
||||
|
||||
def _now():
|
||||
"""本地时间的字符串(到秒),用于时间戳列。"""
|
||||
@@ -38,6 +45,17 @@ def init_db():
|
||||
updated_at TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS task_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
site TEXT,
|
||||
kind TEXT,
|
||||
status TEXT,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
error TEXT
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
@@ -134,3 +152,79 @@ def get_all_status():
|
||||
}
|
||||
for r in rows
|
||||
}
|
||||
|
||||
|
||||
# ============================ 任务历史 ============================
|
||||
|
||||
|
||||
def create_task(site, kind):
|
||||
"""新建一条 pending 任务,返回其 id。"""
|
||||
now = _now()
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO task_history (site, kind, status, started_at, finished_at, error) "
|
||||
"VALUES (?, ?, ?, ?, '', '')",
|
||||
(site, kind, TASK_PENDING, now),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
def update_task(task_id, status, error=None):
|
||||
"""更新任务状态。终态(success/no_data/failed)写入 finished_at。"""
|
||||
finished = _now() if status in (TASK_SUCCESS, TASK_NO_DATA, TASK_FAILED) else ""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
if finished:
|
||||
conn.execute(
|
||||
"UPDATE task_history SET status=?, finished_at=?, error=? WHERE id=?",
|
||||
(status, finished, error or "", task_id),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE task_history SET status=?, error=? WHERE id=?",
|
||||
(status, error or "", task_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_task(task_id):
|
||||
"""返回单条任务 dict,不存在返回 None。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, site, kind, status, started_at, finished_at, error "
|
||||
"FROM task_history WHERE id=?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"id": row[0],
|
||||
"site": row[1],
|
||||
"kind": row[2],
|
||||
"status": row[3],
|
||||
"started_at": row[4],
|
||||
"finished_at": row[5],
|
||||
"error": row[6],
|
||||
}
|
||||
|
||||
|
||||
def list_tasks(limit=20):
|
||||
"""返回最近 limit 条任务(按 id 倒序)。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, site, kind, status, started_at, finished_at, error "
|
||||
"FROM task_history ORDER BY id DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"id": r[0],
|
||||
"site": r[1],
|
||||
"kind": r[2],
|
||||
"status": r[3],
|
||||
"started_at": r[4],
|
||||
"finished_at": r[5],
|
||||
"error": r[6],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user