feat(write_attachments): add --append mode, fix batch timing and SQL Server param limit

- Add --append flag: diff source-table 总排号 against existing
  Common.Attachment SN and classify/write only the missing ones. --limit
  caps the per-run append count, --order sets the direction. Backed by
  new fetch_existing_attachment_sns() in db.py.
- Fix batch elapsed-time accounting: summarize_results() summed each
  task's per-item elapsed_ms, which overcounts under ThreadPoolExecutor
  concurrency (cumulative work time, not real wall-clock — 100 tasks on
  8 workers reported ~5x the actual runtime). Callers now time
  classify_batch() via perf_counter and pass wall_clock_ms; both the
  write_attachments [汇总] line and main.py --summary report wall-clock
  separately from the cumulative sum.
- Format durations >=1s in seconds (88851.4 ms -> 88.85 s) in the
  human-readable [汇总] line; structured JSON --summary fields stay in ms.
- Chunk all IN (...) lists to 2000 items to respect SQL Server's 2100
  bind-parameter hard limit (previously --append --limit 5000 failed at
  the fetch step with "COUNT 字段不正确"). Applied to fetch_params_by_ids
  (sn / id paths) and to the DELETE inside upsert_attachments.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-07-28 11:17:17 +08:00
parent a014a945af
commit 2ae5621090
5 changed files with 166 additions and 42 deletions

99
db.py
View File

@@ -43,11 +43,24 @@ __all__ = [
"fetch_params_by_ids",
"fetch_param_by_id",
"fetch_all_ids",
"fetch_existing_attachment_sns",
"upsert_attachments",
"init_schema",
]
# SQL Server 单条语句最多 2100 个绑定参数WHERE col IN (?, ?, ...) 每项占 1 个,
# 超过会报 "COUNT 字段不正确或语法错误"。这里把超长 IN 列表按 2000 一批分片查询
# 2000 < 2100留安全余量对 PostgreSQL 的 65535 上限也完全够用)。
_PARAM_CHUNK = 2000
def _chunked(seq: list, size: int = _PARAM_CHUNK):
"""把列表切成 <= size 的连续片段,用于分批 IN 查询。"""
for i in range(0, len(seq), size):
yield seq[i : i + size]
def _build_source_table(db_cfg: dict[str, Any]):
"""按配置动态构造源表 Core 结构(带 quote保证跨库标识符正确引用"""
schema = db_cfg["schema"]
@@ -102,14 +115,15 @@ def fetch_params_by_ids(
def _fill_sn() -> None:
if not sn_items:
return
stmt = select(sn_c, param_c).where(sn_c.in_(sn_items))
with engine.connect() as conn:
for row in conn.execute(stmt):
ident = str(row[0])
entry = result.get(ident)
if entry is not None:
entry["param"] = row[1]
entry["sn"] = ident
for chunk in _chunked(sn_items):
stmt = select(sn_c, param_c).where(sn_c.in_(chunk))
for row in conn.execute(stmt):
ident = str(row[0])
entry = result.get(ident)
if entry is not None:
entry["param"] = row[1]
entry["sn"] = ident
def _fill_id() -> None:
if not id_items:
@@ -119,14 +133,15 @@ def fetch_params_by_ids(
logger.warning(
"未配置 database.id_field--id 将退化为按总排号列 [%s] 查询", sn_col
)
stmt = select(sn_c, param_c).where(sn_c.in_(id_items))
with engine.connect() as conn:
for row in conn.execute(stmt):
ident = str(row[0])
entry = result.get(ident)
if entry is not None:
entry["param"] = row[1]
entry["sn"] = ident
for chunk in _chunked(id_items):
stmt = select(sn_c, param_c).where(sn_c.in_(chunk))
for row in conn.execute(stmt):
ident = str(row[0])
entry = result.get(ident)
if entry is not None:
entry["param"] = row[1]
entry["sn"] = ident
return
id_c = src.c[id_field]
@@ -144,17 +159,18 @@ def fetch_params_by_ids(
if not id_ints:
return
stmt = select(id_c, sn_c, param_c).where(id_c.in_(id_ints))
with engine.connect() as conn:
for row in conn.execute(stmt):
idv = row[0]
orig = id_int_map.get(idv)
if orig is None:
continue
entry = result.get(orig)
if entry is not None:
entry["param"] = row[2]
entry["sn"] = row[1]
for chunk in _chunked(id_ints):
stmt = select(id_c, sn_c, param_c).where(id_c.in_(chunk))
for row in conn.execute(stmt):
idv = row[0]
orig = id_int_map.get(idv)
if orig is None:
continue
entry = result.get(orig)
if entry is not None:
entry["param"] = row[2]
entry["sn"] = row[1]
try:
_fill_sn()
@@ -227,6 +243,26 @@ def fetch_all_ids(
raise DatabaseError(f"查询总排号失败: {e}") from e
def fetch_existing_attachment_sns(db_cfg: dict[str, Any]) -> set[str]:
"""读取 Common.Attachment 中已存在的全部 SNdistinct
用于"追加"模式做差集:源表总排号集合 减去 这里返回的 SN 集合,
得到"源表有、但目标表还没写过"的总排号,只对这部分分类写入。
返回 set[str]空表返回空集合SN 统一转成 str便于与源表总排号
(同样是 str直接做集合比较。NULL 值会被过滤掉。
"""
engine = get_engine(db_cfg)
try:
with engine.connect() as conn:
stmt = select(Attachment.SN).distinct()
return {
str(row[0]) for row in conn.execute(stmt) if row[0] is not None
}
except SQLAlchemyError as e:
raise DatabaseError(f"读取已有附件 SN 失败: {e}") from e
def upsert_attachments(
db_cfg: dict[str, Any],
rows: list[tuple[str, str, str]],
@@ -237,6 +273,9 @@ def upsert_attachments(
对出现的每个 SN 先 DELETE 其旧行,再批量 INSERT——保证重跑总是反映
最新分类结果,不会因唯一约束(复合主键 (SN,大类,小类))冲突而失败。
DELETE 的 IN 列表按 _PARAM_CHUNK 分片执行,避免 SN 过多时单条语句超过
SQL Server 的 2100 绑定参数上限;整个"删 + 插"在同一事务内完成,语义不变。
返回 (deleted_rows, inserted_rows) 计数。
"""
if not rows:
@@ -247,10 +286,14 @@ def upsert_attachments(
try:
with Session(engine) as session:
del_res = session.execute(
delete(Attachment).where(Attachment.SN.in_(distinct_sn))
)
deleted = del_res.rowcount if del_res.rowcount is not None else 0
# 分片 DELETE单条 IN (...) 最多 _PARAM_CHUNK 个参数,避免触发 2100 上限
deleted = 0
for chunk in _chunked(distinct_sn):
del_res = session.execute(
delete(Attachment).where(Attachment.SN.in_(chunk))
)
if del_res.rowcount:
deleted += del_res.rowcount
session.execute(
insert(Attachment),
[