feat(state_store): add ingest_state table + set/get helpers
This commit is contained in:
@@ -110,6 +110,17 @@ def init_db():
|
||||
PRIMARY KEY (site, key)
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ingest_state (
|
||||
site TEXT,
|
||||
kind TEXT,
|
||||
ok INTEGER,
|
||||
ingested_at TEXT,
|
||||
count INTEGER,
|
||||
error TEXT,
|
||||
PRIMARY KEY (site, kind)
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
@@ -247,6 +258,39 @@ def get_all_status():
|
||||
}
|
||||
|
||||
|
||||
def set_ingest_state(site, kind, ok, count=0, error=None):
|
||||
"""记录一次入库结果(UPSERT)。ok: bool;count: 入库条数;error: 失败原因或 None。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO ingest_state (site, kind, ok, ingested_at, count, error) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(site, kind) DO UPDATE SET "
|
||||
"ok=excluded.ok, ingested_at=excluded.ingested_at, "
|
||||
"count=excluded.count, error=excluded.error",
|
||||
(site, kind, 1 if ok else 0, _now(), int(count or 0), error or ""),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_all_ingest_state():
|
||||
"""返回 {site: {kind: {ok, ingested_at, count, error}}};库不存在返回 {}。"""
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return {}
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT site, kind, ok, ingested_at, count, error FROM ingest_state"
|
||||
).fetchall()
|
||||
out = {}
|
||||
for site, kind, ok, ingested_at, count, error in rows:
|
||||
out.setdefault(site, {})[kind] = {
|
||||
"ok": bool(ok),
|
||||
"ingested_at": ingested_at or "",
|
||||
"count": int(count or 0),
|
||||
"error": error or "",
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
# ============================ 下载日期偏移(site_config)============================
|
||||
|
||||
MAX_DATE_OFFSET = 30 # 0=今天,最大回溯 30 天
|
||||
|
||||
Reference in New Issue
Block a user