feat(schedule): 周期抓取调度取代每日定点下载

应到/实到数据各自独立周期抓取(启用+激活时段+频率),抓取与处理解耦,抓完自动落库(复用现有 _persist_to_db 钩子)。

- state_store: 新增 fetch_schedule 表(site,kind);所有连接加 timeout=3.0 防并发锁;get/set/get_all_fetch_schedules、create_task_if_idle(周期去重)、allowed_kinds;get_all_config 改返回 fetch_schedules;旧 set_schedule 标废弃。修复 get_all_fetch_schedules 列错位(_fetch_spec(r[2:]))。
- server: IntervalTrigger + _in_active_window(含跨午夜) + _enqueue_fetch(就绪门/激活窗口/去重) + _reschedule_fetch + lifespan 按(site,kind)注册 + FetchScheduleSpec + PUT/GET /config 新结构。
- runtime/store/BFF: 零改动。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka
2026-07-28 22:58:47 +08:00
parent 82ab6d8be3
commit d74a35844f
2 changed files with 261 additions and 70 deletions

View File

@@ -17,11 +17,12 @@ import queue
import threading
import time
from contextlib import asynccontextmanager
from datetime import datetime
from typing import Dict, Optional
import uvicorn
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
from fastapi import FastAPI, HTTPException
from fastapi.responses import FileResponse
from pydantic import BaseModel
@@ -98,46 +99,76 @@ def _worker_loop():
print(">> [worker] 已退出。")
def _enqueue_undelivered(site):
"""定时 job把该站 undelivered 任务投到队列worker 串行处理;本线程不碰 Playwright"""
def _in_active_window(active_start, active_end):
"""当前本地时间是否落在激活时段内(避免半夜空跑)。
- start/end 任一为空 → 不限时段24h 活跃)
- start == end非空→ 视为全天活跃
- start < end → 半开区间 [start, end)
- start > end → 跨午夜(如 22:00-06:00now >= start 或 now < end
"""
if not active_start or not active_end:
return True
now = datetime.now().strftime("%H:%M")
if active_start == active_end:
return True
if active_start < active_end:
return active_start <= now < active_end
return now >= active_start or now < active_end
def _enqueue_fetch(site, kind):
"""周期 job 回调worker 就绪 + 在激活时段内 + 该(site,kind)无未完成任务时,投递一次抓取任务。
跑在 APScheduler 线程池线程——绝不碰 Playwright只经 SQLite + task_queue 通信。"""
if not worker_state["ready"]:
return # worker 未就绪:与 POST /tasks 的 409 同义,下个周期补抓
cfg = state_store.get_fetch_schedule(site, kind)
if not cfg or not cfg["enabled"]:
return # 已禁用job 本应已注销,双重保险)
if not _in_active_window(cfg["active_start"], cfg["active_end"]):
return # 不在激活时段,跳过本次 fire
try:
tid = state_store.create_task(site, "undelivered")
task_queue.put((tid, {"site": site, "kind": "undelivered"}))
print(f">> [定时] 投递 {site}/undelivered 任务 #{tid}")
tid = state_store.create_task_if_idle(site, kind)
if tid is None:
return # 上一次同类任务还没跑完,跳过避免堆积
task_queue.put((tid, {"site": site, "kind": kind}))
print(f">> [周期] 投递 {site}/{kind} 任务 #{tid}")
except Exception as e:
print(f">> [定时] 投递 {site} 失败: {e}")
print(f">> [周期] 投递 {site}/{kind} 失败: {e}")
def _reschedule_site(site):
"""按持久化配置(重新)注册或取消该站的每日定时 job"""
job_id = f"site_{site}"
def _reschedule_fetch(site, kind):
"""按持久化配置(重新)注册或取消该 (site,kind) 的周期抓取 jobIntervalTrigger"""
job_id = f"{site}_{kind}"
try:
scheduler.remove_job(job_id)
except Exception:
pass
cfg = state_store.get_all_config().get(site)
if not cfg or not cfg.get("schedule_enabled") or not cfg.get("schedule_time"):
return
cfg = state_store.get_fetch_schedule(site, kind)
if not cfg or not cfg["enabled"] or cfg["interval_minutes"] < 1:
return # 未启用或频率非法:不注册(等同取消)
try:
hh, mm = cfg["schedule_time"].split(":")
scheduler.add_job(
_enqueue_undelivered,
CronTrigger(hour=int(hh), minute=int(mm)),
args=[site],
_enqueue_fetch,
IntervalTrigger(minutes=cfg["interval_minutes"]),
args=[site, kind],
id=job_id,
replace_existing=True,
)
print(f">> [定时] 已注册 {site} 每日 {cfg['schedule_time']} 下载")
print(
f">> [周期] 已注册 {site}/{kind}:每 {cfg['interval_minutes']} 分钟"
f"(激活 {cfg['active_start'] or '不限'}~{cfg['active_end'] or '不限'}"
)
except Exception as e:
print(f">> [定时] 注册 {site} 失败: {e}")
print(f">> [周期] 注册 {site}/{kind} 失败: {e}")
@asynccontextmanager
async def lifespan(_app):
"""服务启停:起 worker 线程 / 通知 worker 停。"""
state_store.init_db() # 先建表/迁移状态库,确保早于 worker 就绪的 /api/status 可用
for site in ALL_SITES: # 按持久化配置注册各站每日定时 job
_reschedule_site(site)
for site in ALL_SITES: # 按持久化配置注册各站各 kind 的周期抓取 job
for kind in state_store.allowed_kinds(site):
_reschedule_fetch(site, kind)
scheduler.start()
print(">> [定时] 调度器已启动")
t = threading.Thread(target=_worker_loop, daemon=True)
@@ -197,20 +228,28 @@ def get_status():
}
class FetchScheduleSpec(BaseModel):
enabled: bool
active_start: str = ""
active_end: str = ""
interval_minutes: int
class ConfigRequest(BaseModel):
expected_offset: Optional[int] = None
actual_offset: Optional[int] = None
# DEPRECATED旧"每日定点"字段,保留一个发布周期兼容旧前端请求(收到即忽略)
schedule_enabled: Optional[bool] = None
schedule_time: Optional[str] = None
settings: Optional[Dict[str, str]] = None
fetch_schedules: Optional[Dict[str, FetchScheduleSpec]] = None
def _default_cfg():
return {
"expected_offset": 0,
"actual_offset": 0,
"schedule_enabled": False,
"schedule_time": "",
"fetch_schedules": {},
}
@@ -225,7 +264,6 @@ def get_config():
def set_config(site: str, req: ConfigRequest):
if site not in ALL_SITES:
raise HTTPException(status_code=400, detail=f"未知站点: {site}")
cur = state_store.get_all_config().get(site, _default_cfg())
# 应到/实到偏移(百世锁定当天)
for kind, val in (
("expected", req.expected_offset),
@@ -237,18 +275,25 @@ def set_config(site: str, req: ConfigRequest):
status_code=400, detail="百世固定下载当天,不可配置偏移"
)
state_store.set_offset(site, kind, val)
# 每日定时(所有站可配,含百世
if req.schedule_enabled is not None or req.schedule_time is not None:
enabled = (
req.schedule_enabled
if req.schedule_enabled is not None
else cur["schedule_enabled"]
)
time_str = (
req.schedule_time if req.schedule_time is not None else cur["schedule_time"]
)
state_store.set_schedule(site, enabled, time_str)
_reschedule_site(site)
# 周期抓取调度(应到/实到各自独立;百世只允许 undelivered
if req.fetch_schedules:
allowed = state_store.allowed_kinds(site)
for kind, spec in req.fetch_schedules.items():
if kind not in allowed:
raise HTTPException(
status_code=400,
detail=f"站点 {site} 不支持抓取类型 {kind}(允许: {list(allowed)}",
)
interval = max(1, min(1440, int(spec.interval_minutes)))
state_store.set_fetch_schedule(
site,
kind,
spec.enabled,
spec.active_start,
spec.active_end,
interval,
)
_reschedule_fetch(site, kind)
# 站点专属配置(密码/账号/路径…)
if req.settings:
for k, v in req.settings.items():