阶段1:服务化骨架(FastAPI + Playwright worker + 任务队列)

- 新增 runtime.py:抽离共享核心 launch_and_prepare / dispatch_task / run_heartbeat /
  probe_* / launch_anneng / RuntimeContext,常量 SITES_CONFIG 等;main_router 复用
- 新增 server.py:FastAPI(主线程)+ Playwright worker(独立线程)+ 任务队列;
  API:POST/GET /tasks、GET /status、GET /data/{file}(防路径穿越)
- state_store:加 task_history 表 + create/update/get/list 接口
- 5 站点 with_retry 改为返回 True/False,供 dispatch_task 判成败
- main_router:重写为复用 runtime 的交互模式(行为不变)
- requirements:加 fastapi、uvicorn
- CLAUDE.md:补充运行模式与共享核心架构说明

线程模型:主线程 FastAPI 不碰 Playwright,worker 线程独占 page,经 Queue + SQLite 通信。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka
2026-07-17 00:04:39 +08:00
parent a5ceb42ac1
commit 66bd8af421
11 changed files with 882 additions and 493 deletions

View File

@@ -16,6 +16,13 @@ LOGIN_UNKNOWN = "unknown" # 尚未探测过
LOGIN_IN = "logged_in"
LOGIN_OUT = "logged_out"
# 任务状态枚举task_history.status
TASK_PENDING = "pending" # 已入队,待执行
TASK_RUNNING = "running" # 正在执行
TASK_SUCCESS = "success" # 成功(有数据)
TASK_NO_DATA = "no_data" # 成功但本站本次无数据
TASK_FAILED = "failed" # 失败(重试耗尽 / 未登录 / 异常)
def _now():
"""本地时间的字符串(到秒),用于时间戳列。"""
@@ -38,6 +45,17 @@ def init_db():
updated_at TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS task_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
site TEXT,
kind TEXT,
status TEXT,
started_at TEXT,
finished_at TEXT,
error TEXT
)
""")
conn.commit()
@@ -134,3 +152,79 @@ def get_all_status():
}
for r in rows
}
# ============================ 任务历史 ============================
def create_task(site, kind):
"""新建一条 pending 任务,返回其 id。"""
now = _now()
with sqlite3.connect(STATE_DB_PATH) as conn:
cur = conn.execute(
"INSERT INTO task_history (site, kind, status, started_at, finished_at, error) "
"VALUES (?, ?, ?, ?, '', '')",
(site, kind, TASK_PENDING, now),
)
conn.commit()
return cur.lastrowid
def update_task(task_id, status, error=None):
"""更新任务状态。终态(success/no_data/failed)写入 finished_at。"""
finished = _now() if status in (TASK_SUCCESS, TASK_NO_DATA, TASK_FAILED) else ""
with sqlite3.connect(STATE_DB_PATH) as conn:
if finished:
conn.execute(
"UPDATE task_history SET status=?, finished_at=?, error=? WHERE id=?",
(status, finished, error or "", task_id),
)
else:
conn.execute(
"UPDATE task_history SET status=?, error=? WHERE id=?",
(status, error or "", task_id),
)
conn.commit()
def get_task(task_id):
"""返回单条任务 dict不存在返回 None。"""
with sqlite3.connect(STATE_DB_PATH) as conn:
row = conn.execute(
"SELECT id, site, kind, status, started_at, finished_at, error "
"FROM task_history WHERE id=?",
(task_id,),
).fetchone()
if not row:
return None
return {
"id": row[0],
"site": row[1],
"kind": row[2],
"status": row[3],
"started_at": row[4],
"finished_at": row[5],
"error": row[6],
}
def list_tasks(limit=20):
"""返回最近 limit 条任务(按 id 倒序)。"""
with sqlite3.connect(STATE_DB_PATH) as conn:
rows = conn.execute(
"SELECT id, site, kind, status, started_at, finished_at, error "
"FROM task_history ORDER BY id DESC LIMIT ?",
(limit,),
).fetchall()
return [
{
"id": r[0],
"site": r[1],
"kind": r[2],
"status": r[3],
"started_at": r[4],
"finished_at": r[5],
"error": r[6],
}
for r in rows
]