feat(state_store): add ingest_state table + set/get helpers

This commit is contained in:
Misaka_Company
2026-07-24 10:55:43 +08:00
parent 1093256de6
commit 7211846375

View File

@@ -110,6 +110,17 @@ def init_db():
PRIMARY KEY (site, key) PRIMARY KEY (site, key)
) )
""") """)
conn.execute("""
CREATE TABLE IF NOT EXISTS ingest_state (
site TEXT,
kind TEXT,
ok INTEGER,
ingested_at TEXT,
count INTEGER,
error TEXT,
PRIMARY KEY (site, kind)
)
""")
conn.commit() conn.commit()
@@ -247,6 +258,39 @@ def get_all_status():
} }
def set_ingest_state(site, kind, ok, count=0, error=None):
"""记录一次入库结果UPSERT。ok: boolcount: 入库条数error: 失败原因或 None。"""
with sqlite3.connect(STATE_DB_PATH) as conn:
conn.execute(
"INSERT INTO ingest_state (site, kind, ok, ingested_at, count, error) "
"VALUES (?, ?, ?, ?, ?, ?) "
"ON CONFLICT(site, kind) DO UPDATE SET "
"ok=excluded.ok, ingested_at=excluded.ingested_at, "
"count=excluded.count, error=excluded.error",
(site, kind, 1 if ok else 0, _now(), int(count or 0), error or ""),
)
conn.commit()
def get_all_ingest_state():
"""返回 {site: {kind: {ok, ingested_at, count, error}}};库不存在返回 {}"""
if not os.path.exists(STATE_DB_PATH):
return {}
with sqlite3.connect(STATE_DB_PATH) as conn:
rows = conn.execute(
"SELECT site, kind, ok, ingested_at, count, error FROM ingest_state"
).fetchall()
out = {}
for site, kind, ok, ingested_at, count, error in rows:
out.setdefault(site, {})[kind] = {
"ok": bool(ok),
"ingested_at": ingested_at or "",
"count": int(count or 0),
"error": error or "",
}
return out
# ============================ 下载日期偏移site_config============================ # ============================ 下载日期偏移site_config============================
MAX_DATE_OFFSET = 30 # 0=今天,最大回溯 30 天 MAX_DATE_OFFSET = 30 # 0=今天,最大回溯 30 天