feat(schedule): 周期抓取调度取代每日定点下载
应到/实到数据各自独立周期抓取(启用+激活时段+频率),抓取与处理解耦,抓完自动落库(复用现有 _persist_to_db 钩子)。 - state_store: 新增 fetch_schedule 表(site,kind);所有连接加 timeout=3.0 防并发锁;get/set/get_all_fetch_schedules、create_task_if_idle(周期去重)、allowed_kinds;get_all_config 改返回 fetch_schedules;旧 set_schedule 标废弃。修复 get_all_fetch_schedules 列错位(_fetch_spec(r[2:]))。 - server: IntervalTrigger + _in_active_window(含跨午夜) + _enqueue_fetch(就绪门/激活窗口/去重) + _reschedule_fetch + lifespan 按(site,kind)注册 + FetchScheduleSpec + PUT/GET /config 新结构。 - runtime/store/BFF: 零改动。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -17,11 +17,12 @@ import queue
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
from datetime import datetime
|
||||||
from typing import Dict, Optional
|
from typing import Dict, Optional
|
||||||
|
|
||||||
import uvicorn
|
import uvicorn
|
||||||
from apscheduler.schedulers.background import BackgroundScheduler
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
from apscheduler.triggers.cron import CronTrigger
|
from apscheduler.triggers.interval import IntervalTrigger
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
@@ -98,46 +99,76 @@ def _worker_loop():
|
|||||||
print(">> [worker] 已退出。")
|
print(">> [worker] 已退出。")
|
||||||
|
|
||||||
|
|
||||||
def _enqueue_undelivered(site):
|
def _in_active_window(active_start, active_end):
|
||||||
"""定时 job:把该站 undelivered 任务投到队列(worker 串行处理;本线程不碰 Playwright)。"""
|
"""当前本地时间是否落在激活时段内(避免半夜空跑)。
|
||||||
|
- start/end 任一为空 → 不限时段(24h 活跃)
|
||||||
|
- start == end(非空)→ 视为全天活跃
|
||||||
|
- start < end → 半开区间 [start, end)
|
||||||
|
- start > end → 跨午夜(如 22:00-06:00):now >= start 或 now < end
|
||||||
|
"""
|
||||||
|
if not active_start or not active_end:
|
||||||
|
return True
|
||||||
|
now = datetime.now().strftime("%H:%M")
|
||||||
|
if active_start == active_end:
|
||||||
|
return True
|
||||||
|
if active_start < active_end:
|
||||||
|
return active_start <= now < active_end
|
||||||
|
return now >= active_start or now < active_end
|
||||||
|
|
||||||
|
|
||||||
|
def _enqueue_fetch(site, kind):
|
||||||
|
"""周期 job 回调:worker 就绪 + 在激活时段内 + 该(site,kind)无未完成任务时,投递一次抓取任务。
|
||||||
|
跑在 APScheduler 线程池线程——绝不碰 Playwright,只经 SQLite + task_queue 通信。"""
|
||||||
|
if not worker_state["ready"]:
|
||||||
|
return # worker 未就绪:与 POST /tasks 的 409 同义,下个周期补抓
|
||||||
|
cfg = state_store.get_fetch_schedule(site, kind)
|
||||||
|
if not cfg or not cfg["enabled"]:
|
||||||
|
return # 已禁用(job 本应已注销,双重保险)
|
||||||
|
if not _in_active_window(cfg["active_start"], cfg["active_end"]):
|
||||||
|
return # 不在激活时段,跳过本次 fire
|
||||||
try:
|
try:
|
||||||
tid = state_store.create_task(site, "undelivered")
|
tid = state_store.create_task_if_idle(site, kind)
|
||||||
task_queue.put((tid, {"site": site, "kind": "undelivered"}))
|
if tid is None:
|
||||||
print(f">> [定时] 投递 {site}/undelivered 任务 #{tid}")
|
return # 上一次同类任务还没跑完,跳过避免堆积
|
||||||
|
task_queue.put((tid, {"site": site, "kind": kind}))
|
||||||
|
print(f">> [周期] 投递 {site}/{kind} 任务 #{tid}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f">> [定时] 投递 {site} 失败: {e}")
|
print(f">> [周期] 投递 {site}/{kind} 失败: {e}")
|
||||||
|
|
||||||
|
|
||||||
def _reschedule_site(site):
|
def _reschedule_fetch(site, kind):
|
||||||
"""按持久化配置(重新)注册或取消该站的每日定时 job。"""
|
"""按持久化配置(重新)注册或取消该 (site,kind) 的周期抓取 job(IntervalTrigger)。"""
|
||||||
job_id = f"site_{site}"
|
job_id = f"{site}_{kind}"
|
||||||
try:
|
try:
|
||||||
scheduler.remove_job(job_id)
|
scheduler.remove_job(job_id)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
cfg = state_store.get_all_config().get(site)
|
cfg = state_store.get_fetch_schedule(site, kind)
|
||||||
if not cfg or not cfg.get("schedule_enabled") or not cfg.get("schedule_time"):
|
if not cfg or not cfg["enabled"] or cfg["interval_minutes"] < 1:
|
||||||
return
|
return # 未启用或频率非法:不注册(等同取消)
|
||||||
try:
|
try:
|
||||||
hh, mm = cfg["schedule_time"].split(":")
|
|
||||||
scheduler.add_job(
|
scheduler.add_job(
|
||||||
_enqueue_undelivered,
|
_enqueue_fetch,
|
||||||
CronTrigger(hour=int(hh), minute=int(mm)),
|
IntervalTrigger(minutes=cfg["interval_minutes"]),
|
||||||
args=[site],
|
args=[site, kind],
|
||||||
id=job_id,
|
id=job_id,
|
||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
print(f">> [定时] 已注册 {site} 每日 {cfg['schedule_time']} 下载")
|
print(
|
||||||
|
f">> [周期] 已注册 {site}/{kind}:每 {cfg['interval_minutes']} 分钟"
|
||||||
|
f"(激活 {cfg['active_start'] or '不限'}~{cfg['active_end'] or '不限'})"
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f">> [定时] 注册 {site} 失败: {e}")
|
print(f">> [周期] 注册 {site}/{kind} 失败: {e}")
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(_app):
|
async def lifespan(_app):
|
||||||
"""服务启停:起 worker 线程 / 通知 worker 停。"""
|
"""服务启停:起 worker 线程 / 通知 worker 停。"""
|
||||||
state_store.init_db() # 先建表/迁移状态库,确保早于 worker 就绪的 /api/status 可用
|
state_store.init_db() # 先建表/迁移状态库,确保早于 worker 就绪的 /api/status 可用
|
||||||
for site in ALL_SITES: # 按持久化配置注册各站每日定时 job
|
for site in ALL_SITES: # 按持久化配置注册各站各 kind 的周期抓取 job
|
||||||
_reschedule_site(site)
|
for kind in state_store.allowed_kinds(site):
|
||||||
|
_reschedule_fetch(site, kind)
|
||||||
scheduler.start()
|
scheduler.start()
|
||||||
print(">> [定时] 调度器已启动")
|
print(">> [定时] 调度器已启动")
|
||||||
t = threading.Thread(target=_worker_loop, daemon=True)
|
t = threading.Thread(target=_worker_loop, daemon=True)
|
||||||
@@ -197,20 +228,28 @@ def get_status():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FetchScheduleSpec(BaseModel):
|
||||||
|
enabled: bool
|
||||||
|
active_start: str = ""
|
||||||
|
active_end: str = ""
|
||||||
|
interval_minutes: int
|
||||||
|
|
||||||
|
|
||||||
class ConfigRequest(BaseModel):
|
class ConfigRequest(BaseModel):
|
||||||
expected_offset: Optional[int] = None
|
expected_offset: Optional[int] = None
|
||||||
actual_offset: Optional[int] = None
|
actual_offset: Optional[int] = None
|
||||||
|
# DEPRECATED:旧"每日定点"字段,保留一个发布周期兼容旧前端请求(收到即忽略)
|
||||||
schedule_enabled: Optional[bool] = None
|
schedule_enabled: Optional[bool] = None
|
||||||
schedule_time: Optional[str] = None
|
schedule_time: Optional[str] = None
|
||||||
settings: Optional[Dict[str, str]] = None
|
settings: Optional[Dict[str, str]] = None
|
||||||
|
fetch_schedules: Optional[Dict[str, FetchScheduleSpec]] = None
|
||||||
|
|
||||||
|
|
||||||
def _default_cfg():
|
def _default_cfg():
|
||||||
return {
|
return {
|
||||||
"expected_offset": 0,
|
"expected_offset": 0,
|
||||||
"actual_offset": 0,
|
"actual_offset": 0,
|
||||||
"schedule_enabled": False,
|
"fetch_schedules": {},
|
||||||
"schedule_time": "",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -225,7 +264,6 @@ def get_config():
|
|||||||
def set_config(site: str, req: ConfigRequest):
|
def set_config(site: str, req: ConfigRequest):
|
||||||
if site not in ALL_SITES:
|
if site not in ALL_SITES:
|
||||||
raise HTTPException(status_code=400, detail=f"未知站点: {site}")
|
raise HTTPException(status_code=400, detail=f"未知站点: {site}")
|
||||||
cur = state_store.get_all_config().get(site, _default_cfg())
|
|
||||||
# 应到/实到偏移(百世锁定当天)
|
# 应到/实到偏移(百世锁定当天)
|
||||||
for kind, val in (
|
for kind, val in (
|
||||||
("expected", req.expected_offset),
|
("expected", req.expected_offset),
|
||||||
@@ -237,18 +275,25 @@ def set_config(site: str, req: ConfigRequest):
|
|||||||
status_code=400, detail="百世固定下载当天,不可配置偏移"
|
status_code=400, detail="百世固定下载当天,不可配置偏移"
|
||||||
)
|
)
|
||||||
state_store.set_offset(site, kind, val)
|
state_store.set_offset(site, kind, val)
|
||||||
# 每日定时(所有站可配,含百世)
|
# 周期抓取调度(应到/实到各自独立;百世只允许 undelivered)
|
||||||
if req.schedule_enabled is not None or req.schedule_time is not None:
|
if req.fetch_schedules:
|
||||||
enabled = (
|
allowed = state_store.allowed_kinds(site)
|
||||||
req.schedule_enabled
|
for kind, spec in req.fetch_schedules.items():
|
||||||
if req.schedule_enabled is not None
|
if kind not in allowed:
|
||||||
else cur["schedule_enabled"]
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"站点 {site} 不支持抓取类型 {kind}(允许: {list(allowed)})",
|
||||||
)
|
)
|
||||||
time_str = (
|
interval = max(1, min(1440, int(spec.interval_minutes)))
|
||||||
req.schedule_time if req.schedule_time is not None else cur["schedule_time"]
|
state_store.set_fetch_schedule(
|
||||||
|
site,
|
||||||
|
kind,
|
||||||
|
spec.enabled,
|
||||||
|
spec.active_start,
|
||||||
|
spec.active_end,
|
||||||
|
interval,
|
||||||
)
|
)
|
||||||
state_store.set_schedule(site, enabled, time_str)
|
_reschedule_fetch(site, kind)
|
||||||
_reschedule_site(site)
|
|
||||||
# 站点专属配置(密码/账号/路径…)
|
# 站点专属配置(密码/账号/路径…)
|
||||||
if req.settings:
|
if req.settings:
|
||||||
for k, v in req.settings.items():
|
for k, v in req.settings.items():
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ def _now():
|
|||||||
def init_db():
|
def init_db():
|
||||||
"""建库建表(幂等)。确保 state 目录存在。"""
|
"""建库建表(幂等)。确保 state 目录存在。"""
|
||||||
os.makedirs(os.path.dirname(STATE_DB_PATH), exist_ok=True)
|
os.makedirs(os.path.dirname(STATE_DB_PATH), exist_ok=True)
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
conn.execute("""
|
conn.execute("""
|
||||||
CREATE TABLE IF NOT EXISTS site_status (
|
CREATE TABLE IF NOT EXISTS site_status (
|
||||||
site TEXT PRIMARY KEY,
|
site TEXT PRIMARY KEY,
|
||||||
@@ -121,6 +121,21 @@ def init_db():
|
|||||||
PRIMARY KEY (site, kind)
|
PRIMARY KEY (site, kind)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
# 周期性抓取调度(每 site×kind 一行):取代旧 site_config.schedule_* 的每日单时点。
|
||||||
|
# enabled=总开关;active_start/end=激活时段"HH:MM"(空=不限时段,避免半夜空跑);
|
||||||
|
# interval_minutes=激活时段内的抓取间隔。百世 kind 固定为 undelivered(无应到/实到二分)。
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS fetch_schedule (
|
||||||
|
site TEXT NOT NULL,
|
||||||
|
kind TEXT NOT NULL,
|
||||||
|
enabled INTEGER NOT NULL DEFAULT 0,
|
||||||
|
active_start TEXT NOT NULL DEFAULT '',
|
||||||
|
active_end TEXT NOT NULL DEFAULT '',
|
||||||
|
interval_minutes INTEGER NOT NULL DEFAULT 30,
|
||||||
|
updated_at TEXT,
|
||||||
|
PRIMARY KEY (site, kind)
|
||||||
|
)
|
||||||
|
""")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
@@ -202,14 +217,14 @@ def _upsert(conn, site, **fields):
|
|||||||
def set_login_state(site, logged_in):
|
def set_login_state(site, logged_in):
|
||||||
"""更新单站登录态。logged_in: bool。"""
|
"""更新单站登录态。logged_in: bool。"""
|
||||||
state = LOGIN_IN if logged_in else LOGIN_OUT
|
state = LOGIN_IN if logged_in else LOGIN_OUT
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
_upsert(conn, site, login_state=state, login_checked_at=_now())
|
_upsert(conn, site, login_state=state, login_checked_at=_now())
|
||||||
|
|
||||||
|
|
||||||
def reset_login_states(sites):
|
def reset_login_states(sites):
|
||||||
"""启动时把给定站点的登录态重置为 unknown(避免显示上一会话的陈旧登录态)。
|
"""启动时把给定站点的登录态重置为 unknown(避免显示上一会话的陈旧登录态)。
|
||||||
登录态是会话级的;数据态(文件就绪)会话无关、保留不动,心跳就绪后会重新探测。"""
|
登录态是会话级的;数据态(文件就绪)会话无关、保留不动,心跳就绪后会重新探测。"""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
for site in sites:
|
for site in sites:
|
||||||
_upsert(conn, site, login_state=LOGIN_UNKNOWN, login_checked_at="")
|
_upsert(conn, site, login_state=LOGIN_UNKNOWN, login_checked_at="")
|
||||||
|
|
||||||
@@ -223,7 +238,7 @@ def set_data_state(site, kind, ready, generated_at, business_date=None):
|
|||||||
}
|
}
|
||||||
if business_date is not None:
|
if business_date is not None:
|
||||||
fields[f"{kind}_business_date"] = business_date or ""
|
fields[f"{kind}_business_date"] = business_date or ""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
_upsert(conn, site, **fields)
|
_upsert(conn, site, **fields)
|
||||||
|
|
||||||
|
|
||||||
@@ -231,7 +246,7 @@ def get_all_status():
|
|||||||
"""返回 {site: {各字段}};库不存在则返回 {}。"""
|
"""返回 {site: {各字段}};库不存在则返回 {}。"""
|
||||||
if not os.path.exists(STATE_DB_PATH):
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
return {}
|
return {}
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT site, login_state, login_checked_at, expected_ready, "
|
"SELECT site, login_state, login_checked_at, expected_ready, "
|
||||||
"expected_generated_at, expected_business_date, actual_ready, "
|
"expected_generated_at, expected_business_date, actual_ready, "
|
||||||
@@ -260,7 +275,7 @@ def get_all_status():
|
|||||||
|
|
||||||
def set_ingest_state(site, kind, ok, count=0, error=None):
|
def set_ingest_state(site, kind, ok, count=0, error=None):
|
||||||
"""记录一次入库结果(UPSERT)。ok: bool;count: 入库条数;error: 失败原因或 None。"""
|
"""记录一次入库结果(UPSERT)。ok: bool;count: 入库条数;error: 失败原因或 None。"""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO ingest_state (site, kind, ok, ingested_at, count, error) "
|
"INSERT INTO ingest_state (site, kind, ok, ingested_at, count, error) "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?) "
|
"VALUES (?, ?, ?, ?, ?, ?) "
|
||||||
@@ -276,7 +291,7 @@ def get_all_ingest_state():
|
|||||||
"""返回 {site: {kind: {ok, ingested_at, count, error}}};库不存在返回 {}。"""
|
"""返回 {site: {kind: {ok, ingested_at, count, error}}};库不存在返回 {}。"""
|
||||||
if not os.path.exists(STATE_DB_PATH):
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
return {}
|
return {}
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT site, kind, ok, ingested_at, count, error FROM ingest_state"
|
"SELECT site, kind, ok, ingested_at, count, error FROM ingest_state"
|
||||||
).fetchall()
|
).fetchall()
|
||||||
@@ -301,7 +316,7 @@ def get_offset(site, kind="expected"):
|
|||||||
col = "expected_offset" if kind == "expected" else "actual_offset"
|
col = "expected_offset" if kind == "expected" else "actual_offset"
|
||||||
if not os.path.exists(STATE_DB_PATH):
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
return 0
|
return 0
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
f"SELECT {col} FROM site_config WHERE site=?", (site,)
|
f"SELECT {col} FROM site_config WHERE site=?", (site,)
|
||||||
).fetchone()
|
).fetchone()
|
||||||
@@ -312,7 +327,7 @@ def set_offset(site, kind, offset):
|
|||||||
"""设置单站下载日期偏移(kind: 'expected'/'actual'),钳制到 [0, MAX_DATE_OFFSET]。"""
|
"""设置单站下载日期偏移(kind: 'expected'/'actual'),钳制到 [0, MAX_DATE_OFFSET]。"""
|
||||||
col = "expected_offset" if kind == "expected" else "actual_offset"
|
col = "expected_offset" if kind == "expected" else "actual_offset"
|
||||||
offset = max(0, min(MAX_DATE_OFFSET, int(offset)))
|
offset = max(0, min(MAX_DATE_OFFSET, int(offset)))
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
f"INSERT INTO site_config (site, {col}, updated_at) VALUES (?, ?, ?) "
|
f"INSERT INTO site_config (site, {col}, updated_at) VALUES (?, ?, ?) "
|
||||||
f"ON CONFLICT(site) DO UPDATE SET {col}=excluded.{col}, "
|
f"ON CONFLICT(site) DO UPDATE SET {col}=excluded.{col}, "
|
||||||
@@ -324,9 +339,10 @@ def set_offset(site, kind, offset):
|
|||||||
|
|
||||||
|
|
||||||
def set_schedule(site, enabled, time_str):
|
def set_schedule(site, enabled, time_str):
|
||||||
"""设置单站每日定时下载(enabled: bool;time_str: 'HH:MM' 或 '')。"""
|
"""【DEPRECATED】旧"每日单时点定时"——已被 fetch_schedule 的周期+激活时段模式取代。
|
||||||
|
保留死代码以防外部残留调用;新代码请用 set_fetch_schedule。"""
|
||||||
enabled_int = 1 if enabled else 0
|
enabled_int = 1 if enabled else 0
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO site_config (site, schedule_enabled, schedule_time, updated_at) "
|
"INSERT INTO site_config (site, schedule_enabled, schedule_time, updated_at) "
|
||||||
"VALUES (?, ?, ?, ?) "
|
"VALUES (?, ?, ?, ?) "
|
||||||
@@ -339,24 +355,129 @@ def set_schedule(site, enabled, time_str):
|
|||||||
return bool(enabled_int), (time_str or "")
|
return bool(enabled_int), (time_str or "")
|
||||||
|
|
||||||
|
|
||||||
def get_all_config():
|
# ============================ 周期性抓取调度(fetch_schedule)============================
|
||||||
"""返回 {site: {expected_offset, actual_offset, schedule_enabled, schedule_time}}。"""
|
# 取代旧 site_config.schedule_* 的"每日单时点":每 site×kind 一行,
|
||||||
if not os.path.exists(STATE_DB_PATH):
|
# 在激活时段 [active_start, active_end) 内按 interval_minutes 周期抓取。
|
||||||
return {}
|
# 应到/实到各自独立配置;百世只有 undelivered(站点直供未到明细,无应到/实到二分)。
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
|
||||||
rows = conn.execute(
|
# 各站允许的周期抓取 kind(单一来源,server.py 复用)
|
||||||
"SELECT site, expected_offset, actual_offset, schedule_enabled, schedule_time "
|
SITE_FETCH_KINDS = {
|
||||||
"FROM site_config"
|
"顺心": ("expected", "actual"),
|
||||||
).fetchall()
|
"中通": ("expected", "actual"),
|
||||||
return {
|
"韵达": ("expected", "actual"),
|
||||||
r[0]: {
|
"安能": ("expected", "actual"),
|
||||||
"expected_offset": int(r[1]),
|
"百世": ("undelivered",),
|
||||||
"actual_offset": int(r[2]),
|
|
||||||
"schedule_enabled": bool(r[3]),
|
|
||||||
"schedule_time": r[4] or "",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DEFAULT_FETCH_SCHEDULE = {
|
||||||
|
"enabled": False,
|
||||||
|
"active_start": "",
|
||||||
|
"active_end": "",
|
||||||
|
"interval_minutes": 30,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def allowed_kinds(site):
|
||||||
|
"""该站允许的周期抓取 kind 元组;未知站点返回空元组。"""
|
||||||
|
return SITE_FETCH_KINDS.get(site, ())
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_spec(row):
|
||||||
|
"""把 fetch_schedule 行转成 spec dict。"""
|
||||||
|
return {
|
||||||
|
"enabled": bool(row[0]),
|
||||||
|
"active_start": row[1] or "",
|
||||||
|
"active_end": row[2] or "",
|
||||||
|
"interval_minutes": int(row[3]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_fetch_schedule(site, kind):
|
||||||
|
"""读单 (site,kind) 周期抓取配置;未配置返回 None。"""
|
||||||
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
|
return None
|
||||||
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT enabled, active_start, active_end, interval_minutes "
|
||||||
|
"FROM fetch_schedule WHERE site=? AND kind=?",
|
||||||
|
(site, kind),
|
||||||
|
).fetchone()
|
||||||
|
return _fetch_spec(row) if row else None
|
||||||
|
|
||||||
|
|
||||||
|
def set_fetch_schedule(site, kind, enabled, active_start, active_end, interval_minutes):
|
||||||
|
"""UPSERT 单 (site,kind) 周期抓取配置;返回写入后的 spec dict。"""
|
||||||
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO fetch_schedule "
|
||||||
|
"(site, kind, enabled, active_start, active_end, interval_minutes, updated_at) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?, ?) "
|
||||||
|
"ON CONFLICT(site, kind) DO UPDATE SET "
|
||||||
|
"enabled=excluded.enabled, active_start=excluded.active_start, "
|
||||||
|
"active_end=excluded.active_end, interval_minutes=excluded.interval_minutes, "
|
||||||
|
"updated_at=excluded.updated_at",
|
||||||
|
(
|
||||||
|
site,
|
||||||
|
kind,
|
||||||
|
1 if enabled else 0,
|
||||||
|
(active_start or ""),
|
||||||
|
(active_end or ""),
|
||||||
|
int(interval_minutes),
|
||||||
|
_now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return {
|
||||||
|
"enabled": bool(enabled),
|
||||||
|
"active_start": active_start or "",
|
||||||
|
"active_end": active_end or "",
|
||||||
|
"interval_minutes": int(interval_minutes),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_fetch_schedules():
|
||||||
|
"""返回 {site: {kind: spec}};对每个站点的每个 allowed kind 都补齐(缺失用默认值)。
|
||||||
|
保证前端永远拿到完整 kind 集,不必前端补默认 spec。"""
|
||||||
|
out = {}
|
||||||
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
|
for site, kinds in SITE_FETCH_KINDS.items():
|
||||||
|
out[site] = {k: dict(DEFAULT_FETCH_SCHEDULE) for k in kinds}
|
||||||
|
return out
|
||||||
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT site, kind, enabled, active_start, active_end, interval_minutes "
|
||||||
|
"FROM fetch_schedule"
|
||||||
|
).fetchall()
|
||||||
|
by_key = {(r[0], r[1]): _fetch_spec(r[2:]) for r in rows}
|
||||||
|
for site, kinds in SITE_FETCH_KINDS.items():
|
||||||
|
out[site] = {
|
||||||
|
k: by_key.get((site, k), dict(DEFAULT_FETCH_SCHEDULE)) for k in kinds
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_config():
|
||||||
|
"""返回 {site: {expected_offset, actual_offset, fetch_schedules}}。
|
||||||
|
站点集以 fetch_schedule 的 allowed kinds 为准(覆盖全业务站点);offsets 缺失默认 0。"""
|
||||||
|
schedules = get_all_fetch_schedules()
|
||||||
|
offsets = {}
|
||||||
|
if os.path.exists(STATE_DB_PATH):
|
||||||
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT site, expected_offset, actual_offset FROM site_config"
|
||||||
|
).fetchall()
|
||||||
|
offsets = {
|
||||||
|
r[0]: {"expected_offset": int(r[1]), "actual_offset": int(r[2])}
|
||||||
for r in rows
|
for r in rows
|
||||||
}
|
}
|
||||||
|
return {
|
||||||
|
site: {
|
||||||
|
"expected_offset": offsets.get(site, {}).get("expected_offset", 0),
|
||||||
|
"actual_offset": offsets.get(site, {}).get("actual_offset", 0),
|
||||||
|
"fetch_schedules": schedules.get(site, {}),
|
||||||
|
}
|
||||||
|
for site in schedules
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ============================ 站点键值配置(site_settings)============================
|
# ============================ 站点键值配置(site_settings)============================
|
||||||
@@ -367,7 +488,7 @@ def get_setting(site, key):
|
|||||||
"""读取单站某个配置值;未设置返回 ''。"""
|
"""读取单站某个配置值;未设置返回 ''。"""
|
||||||
if not os.path.exists(STATE_DB_PATH):
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
return ""
|
return ""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
"SELECT value FROM site_settings WHERE site=? AND key=?", (site, key)
|
"SELECT value FROM site_settings WHERE site=? AND key=?", (site, key)
|
||||||
).fetchone()
|
).fetchone()
|
||||||
@@ -376,7 +497,7 @@ def get_setting(site, key):
|
|||||||
|
|
||||||
def set_setting(site, key, value):
|
def set_setting(site, key, value):
|
||||||
"""设置单站某个配置值(upsert)。"""
|
"""设置单站某个配置值(upsert)。"""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO site_settings (site, key, value) VALUES (?, ?, ?) "
|
"INSERT INTO site_settings (site, key, value) VALUES (?, ?, ?) "
|
||||||
"ON CONFLICT(site, key) DO UPDATE SET value=excluded.value",
|
"ON CONFLICT(site, key) DO UPDATE SET value=excluded.value",
|
||||||
@@ -389,7 +510,7 @@ def get_site_settings(site):
|
|||||||
"""返回单站全部配置 {key: value}。"""
|
"""返回单站全部配置 {key: value}。"""
|
||||||
if not os.path.exists(STATE_DB_PATH):
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
return {}
|
return {}
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT key, value FROM site_settings WHERE site=?", (site,)
|
"SELECT key, value FROM site_settings WHERE site=?", (site,)
|
||||||
).fetchall()
|
).fetchall()
|
||||||
@@ -402,7 +523,32 @@ def get_site_settings(site):
|
|||||||
def create_task(site, kind):
|
def create_task(site, kind):
|
||||||
"""新建一条 pending 任务,返回其 id。"""
|
"""新建一条 pending 任务,返回其 id。"""
|
||||||
now = _now()
|
now = _now()
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
|
cur = conn.execute(
|
||||||
|
"INSERT INTO task_history (site, kind, status, started_at, finished_at, error) "
|
||||||
|
"VALUES (?, ?, ?, ?, '', '')",
|
||||||
|
(site, kind, TASK_PENDING, now),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return cur.lastrowid
|
||||||
|
|
||||||
|
|
||||||
|
def create_task_if_idle(site, kind):
|
||||||
|
"""周期调度专用:若该 (site,kind) 已有 pending/running 任务则返回 None(跳过本次周期),
|
||||||
|
否则建一条 pending 任务返回其 id。单连接内 check-then-insert,靠 SQLite 写锁把竞态压到忽略不计。
|
||||||
|
|
||||||
|
与 create_task 的区别:手动触发(POST /tasks)用 create_task(用户点的必建);周期 job 用本函数
|
||||||
|
——上一次还没跑完时跳过,避免同 (site,kind) 任务堆积。手动建的任务会让紧随其后的周期 fire
|
||||||
|
判到 inflight 而跳过,天然互斥。"""
|
||||||
|
now = _now()
|
||||||
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT 1 FROM task_history WHERE site=? AND kind=? "
|
||||||
|
"AND status IN (?, ?) LIMIT 1",
|
||||||
|
(site, kind, TASK_PENDING, TASK_RUNNING),
|
||||||
|
).fetchone()
|
||||||
|
if row:
|
||||||
|
return None
|
||||||
cur = conn.execute(
|
cur = conn.execute(
|
||||||
"INSERT INTO task_history (site, kind, status, started_at, finished_at, error) "
|
"INSERT INTO task_history (site, kind, status, started_at, finished_at, error) "
|
||||||
"VALUES (?, ?, ?, ?, '', '')",
|
"VALUES (?, ?, ?, ?, '', '')",
|
||||||
@@ -415,7 +561,7 @@ def create_task(site, kind):
|
|||||||
def update_task(task_id, status, error=None):
|
def update_task(task_id, status, error=None):
|
||||||
"""更新任务状态。终态(success/no_data/failed)写入 finished_at。"""
|
"""更新任务状态。终态(success/no_data/failed)写入 finished_at。"""
|
||||||
finished = _now() if status in (TASK_SUCCESS, TASK_NO_DATA, TASK_FAILED) else ""
|
finished = _now() if status in (TASK_SUCCESS, TASK_NO_DATA, TASK_FAILED) else ""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
if finished:
|
if finished:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE task_history SET status=?, finished_at=?, error=? WHERE id=?",
|
"UPDATE task_history SET status=?, finished_at=?, error=? WHERE id=?",
|
||||||
@@ -431,7 +577,7 @@ def update_task(task_id, status, error=None):
|
|||||||
|
|
||||||
def get_task(task_id):
|
def get_task(task_id):
|
||||||
"""返回单条任务 dict,不存在返回 None。"""
|
"""返回单条任务 dict,不存在返回 None。"""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
"SELECT id, site, kind, status, started_at, finished_at, error "
|
"SELECT id, site, kind, status, started_at, finished_at, error "
|
||||||
"FROM task_history WHERE id=?",
|
"FROM task_history WHERE id=?",
|
||||||
@@ -452,7 +598,7 @@ def get_task(task_id):
|
|||||||
|
|
||||||
def list_tasks(limit=20):
|
def list_tasks(limit=20):
|
||||||
"""返回最近 limit 条任务(按 id 倒序)。"""
|
"""返回最近 limit 条任务(按 id 倒序)。"""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT id, site, kind, status, started_at, finished_at, error "
|
"SELECT id, site, kind, status, started_at, finished_at, error "
|
||||||
"FROM task_history ORDER BY id DESC LIMIT ?",
|
"FROM task_history ORDER BY id DESC LIMIT ?",
|
||||||
@@ -475,7 +621,7 @@ def list_tasks(limit=20):
|
|||||||
def fail_stale_tasks(reason: str = "服务重启,上轮未完成任务,请手动重跑") -> int:
|
def fail_stale_tasks(reason: str = "服务重启,上轮未完成任务,请手动重跑") -> int:
|
||||||
"""worker 启动时调用:把遗留的 pending/running 任务标记为 failed,实现重启自愈。
|
"""worker 启动时调用:把遗留的 pending/running 任务标记为 failed,实现重启自愈。
|
||||||
返回被清理的任务数量。"""
|
返回被清理的任务数量。"""
|
||||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||||
cur = conn.execute(
|
cur = conn.execute(
|
||||||
"UPDATE task_history SET status=?, finished_at=?, error=? "
|
"UPDATE task_history SET status=?, finished_at=?, error=? "
|
||||||
"WHERE status IN (?, ?)",
|
"WHERE status IN (?, ?)",
|
||||||
|
|||||||
Reference in New Issue
Block a user