feat(schedule): 周期抓取调度取代每日定点下载
应到/实到数据各自独立周期抓取(启用+激活时段+频率),抓取与处理解耦,抓完自动落库(复用现有 _persist_to_db 钩子)。 - state_store: 新增 fetch_schedule 表(site,kind);所有连接加 timeout=3.0 防并发锁;get/set/get_all_fetch_schedules、create_task_if_idle(周期去重)、allowed_kinds;get_all_config 改返回 fetch_schedules;旧 set_schedule 标废弃。修复 get_all_fetch_schedules 列错位(_fetch_spec(r[2:]))。 - server: IntervalTrigger + _in_active_window(含跨午夜) + _enqueue_fetch(就绪门/激活窗口/去重) + _reschedule_fetch + lifespan 按(site,kind)注册 + FetchScheduleSpec + PUT/GET /config 新结构。 - runtime/store/BFF: 零改动。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -17,11 +17,12 @@ import queue
|
||||
import threading
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
import uvicorn
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
@@ -98,46 +99,76 @@ def _worker_loop():
|
||||
print(">> [worker] 已退出。")
|
||||
|
||||
|
||||
def _enqueue_undelivered(site):
|
||||
"""定时 job:把该站 undelivered 任务投到队列(worker 串行处理;本线程不碰 Playwright)。"""
|
||||
def _in_active_window(active_start, active_end):
|
||||
"""当前本地时间是否落在激活时段内(避免半夜空跑)。
|
||||
- start/end 任一为空 → 不限时段(24h 活跃)
|
||||
- start == end(非空)→ 视为全天活跃
|
||||
- start < end → 半开区间 [start, end)
|
||||
- start > end → 跨午夜(如 22:00-06:00):now >= start 或 now < end
|
||||
"""
|
||||
if not active_start or not active_end:
|
||||
return True
|
||||
now = datetime.now().strftime("%H:%M")
|
||||
if active_start == active_end:
|
||||
return True
|
||||
if active_start < active_end:
|
||||
return active_start <= now < active_end
|
||||
return now >= active_start or now < active_end
|
||||
|
||||
|
||||
def _enqueue_fetch(site, kind):
|
||||
"""周期 job 回调:worker 就绪 + 在激活时段内 + 该(site,kind)无未完成任务时,投递一次抓取任务。
|
||||
跑在 APScheduler 线程池线程——绝不碰 Playwright,只经 SQLite + task_queue 通信。"""
|
||||
if not worker_state["ready"]:
|
||||
return # worker 未就绪:与 POST /tasks 的 409 同义,下个周期补抓
|
||||
cfg = state_store.get_fetch_schedule(site, kind)
|
||||
if not cfg or not cfg["enabled"]:
|
||||
return # 已禁用(job 本应已注销,双重保险)
|
||||
if not _in_active_window(cfg["active_start"], cfg["active_end"]):
|
||||
return # 不在激活时段,跳过本次 fire
|
||||
try:
|
||||
tid = state_store.create_task(site, "undelivered")
|
||||
task_queue.put((tid, {"site": site, "kind": "undelivered"}))
|
||||
print(f">> [定时] 投递 {site}/undelivered 任务 #{tid}")
|
||||
tid = state_store.create_task_if_idle(site, kind)
|
||||
if tid is None:
|
||||
return # 上一次同类任务还没跑完,跳过避免堆积
|
||||
task_queue.put((tid, {"site": site, "kind": kind}))
|
||||
print(f">> [周期] 投递 {site}/{kind} 任务 #{tid}")
|
||||
except Exception as e:
|
||||
print(f">> [定时] 投递 {site} 失败: {e}")
|
||||
print(f">> [周期] 投递 {site}/{kind} 失败: {e}")
|
||||
|
||||
|
||||
def _reschedule_site(site):
|
||||
"""按持久化配置(重新)注册或取消该站的每日定时 job。"""
|
||||
job_id = f"site_{site}"
|
||||
def _reschedule_fetch(site, kind):
|
||||
"""按持久化配置(重新)注册或取消该 (site,kind) 的周期抓取 job(IntervalTrigger)。"""
|
||||
job_id = f"{site}_{kind}"
|
||||
try:
|
||||
scheduler.remove_job(job_id)
|
||||
except Exception:
|
||||
pass
|
||||
cfg = state_store.get_all_config().get(site)
|
||||
if not cfg or not cfg.get("schedule_enabled") or not cfg.get("schedule_time"):
|
||||
return
|
||||
cfg = state_store.get_fetch_schedule(site, kind)
|
||||
if not cfg or not cfg["enabled"] or cfg["interval_minutes"] < 1:
|
||||
return # 未启用或频率非法:不注册(等同取消)
|
||||
try:
|
||||
hh, mm = cfg["schedule_time"].split(":")
|
||||
scheduler.add_job(
|
||||
_enqueue_undelivered,
|
||||
CronTrigger(hour=int(hh), minute=int(mm)),
|
||||
args=[site],
|
||||
_enqueue_fetch,
|
||||
IntervalTrigger(minutes=cfg["interval_minutes"]),
|
||||
args=[site, kind],
|
||||
id=job_id,
|
||||
replace_existing=True,
|
||||
)
|
||||
print(f">> [定时] 已注册 {site} 每日 {cfg['schedule_time']} 下载")
|
||||
print(
|
||||
f">> [周期] 已注册 {site}/{kind}:每 {cfg['interval_minutes']} 分钟"
|
||||
f"(激活 {cfg['active_start'] or '不限'}~{cfg['active_end'] or '不限'})"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f">> [定时] 注册 {site} 失败: {e}")
|
||||
print(f">> [周期] 注册 {site}/{kind} 失败: {e}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app):
|
||||
"""服务启停:起 worker 线程 / 通知 worker 停。"""
|
||||
state_store.init_db() # 先建表/迁移状态库,确保早于 worker 就绪的 /api/status 可用
|
||||
for site in ALL_SITES: # 按持久化配置注册各站每日定时 job
|
||||
_reschedule_site(site)
|
||||
for site in ALL_SITES: # 按持久化配置注册各站各 kind 的周期抓取 job
|
||||
for kind in state_store.allowed_kinds(site):
|
||||
_reschedule_fetch(site, kind)
|
||||
scheduler.start()
|
||||
print(">> [定时] 调度器已启动")
|
||||
t = threading.Thread(target=_worker_loop, daemon=True)
|
||||
@@ -197,20 +228,28 @@ def get_status():
|
||||
}
|
||||
|
||||
|
||||
class FetchScheduleSpec(BaseModel):
|
||||
enabled: bool
|
||||
active_start: str = ""
|
||||
active_end: str = ""
|
||||
interval_minutes: int
|
||||
|
||||
|
||||
class ConfigRequest(BaseModel):
|
||||
expected_offset: Optional[int] = None
|
||||
actual_offset: Optional[int] = None
|
||||
# DEPRECATED:旧"每日定点"字段,保留一个发布周期兼容旧前端请求(收到即忽略)
|
||||
schedule_enabled: Optional[bool] = None
|
||||
schedule_time: Optional[str] = None
|
||||
settings: Optional[Dict[str, str]] = None
|
||||
fetch_schedules: Optional[Dict[str, FetchScheduleSpec]] = None
|
||||
|
||||
|
||||
def _default_cfg():
|
||||
return {
|
||||
"expected_offset": 0,
|
||||
"actual_offset": 0,
|
||||
"schedule_enabled": False,
|
||||
"schedule_time": "",
|
||||
"fetch_schedules": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -225,7 +264,6 @@ def get_config():
|
||||
def set_config(site: str, req: ConfigRequest):
|
||||
if site not in ALL_SITES:
|
||||
raise HTTPException(status_code=400, detail=f"未知站点: {site}")
|
||||
cur = state_store.get_all_config().get(site, _default_cfg())
|
||||
# 应到/实到偏移(百世锁定当天)
|
||||
for kind, val in (
|
||||
("expected", req.expected_offset),
|
||||
@@ -237,18 +275,25 @@ def set_config(site: str, req: ConfigRequest):
|
||||
status_code=400, detail="百世固定下载当天,不可配置偏移"
|
||||
)
|
||||
state_store.set_offset(site, kind, val)
|
||||
# 每日定时(所有站可配,含百世)
|
||||
if req.schedule_enabled is not None or req.schedule_time is not None:
|
||||
enabled = (
|
||||
req.schedule_enabled
|
||||
if req.schedule_enabled is not None
|
||||
else cur["schedule_enabled"]
|
||||
)
|
||||
time_str = (
|
||||
req.schedule_time if req.schedule_time is not None else cur["schedule_time"]
|
||||
)
|
||||
state_store.set_schedule(site, enabled, time_str)
|
||||
_reschedule_site(site)
|
||||
# 周期抓取调度(应到/实到各自独立;百世只允许 undelivered)
|
||||
if req.fetch_schedules:
|
||||
allowed = state_store.allowed_kinds(site)
|
||||
for kind, spec in req.fetch_schedules.items():
|
||||
if kind not in allowed:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"站点 {site} 不支持抓取类型 {kind}(允许: {list(allowed)})",
|
||||
)
|
||||
interval = max(1, min(1440, int(spec.interval_minutes)))
|
||||
state_store.set_fetch_schedule(
|
||||
site,
|
||||
kind,
|
||||
spec.enabled,
|
||||
spec.active_start,
|
||||
spec.active_end,
|
||||
interval,
|
||||
)
|
||||
_reschedule_fetch(site, kind)
|
||||
# 站点专属配置(密码/账号/路径…)
|
||||
if req.settings:
|
||||
for k, v in req.settings.items():
|
||||
|
||||
Reference in New Issue
Block a user