feat(state_store): add ingest_state table + set/get helpers
This commit is contained in:
@@ -110,6 +110,17 @@ def init_db():
|
|||||||
PRIMARY KEY (site, key)
|
PRIMARY KEY (site, key)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS ingest_state (
|
||||||
|
site TEXT,
|
||||||
|
kind TEXT,
|
||||||
|
ok INTEGER,
|
||||||
|
ingested_at TEXT,
|
||||||
|
count INTEGER,
|
||||||
|
error TEXT,
|
||||||
|
PRIMARY KEY (site, kind)
|
||||||
|
)
|
||||||
|
""")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
@@ -247,6 +258,39 @@ def get_all_status():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def set_ingest_state(site, kind, ok, count=0, error=None):
|
||||||
|
"""记录一次入库结果(UPSERT)。ok: bool;count: 入库条数;error: 失败原因或 None。"""
|
||||||
|
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO ingest_state (site, kind, ok, ingested_at, count, error) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?) "
|
||||||
|
"ON CONFLICT(site, kind) DO UPDATE SET "
|
||||||
|
"ok=excluded.ok, ingested_at=excluded.ingested_at, "
|
||||||
|
"count=excluded.count, error=excluded.error",
|
||||||
|
(site, kind, 1 if ok else 0, _now(), int(count or 0), error or ""),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_ingest_state():
|
||||||
|
"""返回 {site: {kind: {ok, ingested_at, count, error}}};库不存在返回 {}。"""
|
||||||
|
if not os.path.exists(STATE_DB_PATH):
|
||||||
|
return {}
|
||||||
|
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT site, kind, ok, ingested_at, count, error FROM ingest_state"
|
||||||
|
).fetchall()
|
||||||
|
out = {}
|
||||||
|
for site, kind, ok, ingested_at, count, error in rows:
|
||||||
|
out.setdefault(site, {})[kind] = {
|
||||||
|
"ok": bool(ok),
|
||||||
|
"ingested_at": ingested_at or "",
|
||||||
|
"count": int(count or 0),
|
||||||
|
"error": error or "",
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
# ============================ 下载日期偏移(site_config)============================
|
# ============================ 下载日期偏移(site_config)============================
|
||||||
|
|
||||||
MAX_DATE_OFFSET = 30 # 0=今天,最大回溯 30 天
|
MAX_DATE_OFFSET = 30 # 0=今天,最大回溯 30 天
|
||||||
|
|||||||
Reference in New Issue
Block a user