feat(write_attachments): add --append mode, fix batch timing and SQL Server param limit
- Add --append flag: diff source-table 总排号 against existing Common.Attachment SN and classify/write only the missing ones. --limit caps the per-run append count, --order sets the direction. Backed by new fetch_existing_attachment_sns() in db.py. - Fix batch elapsed-time accounting: summarize_results() summed each task's per-item elapsed_ms, which overcounts under ThreadPoolExecutor concurrency (cumulative work time, not real wall-clock — 100 tasks on 8 workers reported ~5x the actual runtime). Callers now time classify_batch() via perf_counter and pass wall_clock_ms; both the write_attachments [汇总] line and main.py --summary report wall-clock separately from the cumulative sum. - Format durations >=1s in seconds (88851.4 ms -> 88.85 s) in the human-readable [汇总] line; structured JSON --summary fields stay in ms. - Chunk all IN (...) lists to 2000 items to respect SQL Server's 2100 bind-parameter hard limit (previously --append --limit 5000 failed at the fetch step with "COUNT 字段不正确"). Applied to fetch_params_by_ids (sn / id paths) and to the DELETE inside upsert_attachments. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
99
db.py
99
db.py
@@ -43,11 +43,24 @@ __all__ = [
|
||||
"fetch_params_by_ids",
|
||||
"fetch_param_by_id",
|
||||
"fetch_all_ids",
|
||||
"fetch_existing_attachment_sns",
|
||||
"upsert_attachments",
|
||||
"init_schema",
|
||||
]
|
||||
|
||||
|
||||
# SQL Server 单条语句最多 2100 个绑定参数,WHERE col IN (?, ?, ...) 每项占 1 个,
|
||||
# 超过会报 "COUNT 字段不正确或语法错误"。这里把超长 IN 列表按 2000 一批分片查询
|
||||
# (2000 < 2100,留安全余量;对 PostgreSQL 的 65535 上限也完全够用)。
|
||||
_PARAM_CHUNK = 2000
|
||||
|
||||
|
||||
def _chunked(seq: list, size: int = _PARAM_CHUNK):
|
||||
"""把列表切成 <= size 的连续片段,用于分批 IN 查询。"""
|
||||
for i in range(0, len(seq), size):
|
||||
yield seq[i : i + size]
|
||||
|
||||
|
||||
def _build_source_table(db_cfg: dict[str, Any]):
|
||||
"""按配置动态构造源表 Core 结构(带 quote,保证跨库标识符正确引用)。"""
|
||||
schema = db_cfg["schema"]
|
||||
@@ -102,14 +115,15 @@ def fetch_params_by_ids(
|
||||
def _fill_sn() -> None:
|
||||
if not sn_items:
|
||||
return
|
||||
stmt = select(sn_c, param_c).where(sn_c.in_(sn_items))
|
||||
with engine.connect() as conn:
|
||||
for row in conn.execute(stmt):
|
||||
ident = str(row[0])
|
||||
entry = result.get(ident)
|
||||
if entry is not None:
|
||||
entry["param"] = row[1]
|
||||
entry["sn"] = ident
|
||||
for chunk in _chunked(sn_items):
|
||||
stmt = select(sn_c, param_c).where(sn_c.in_(chunk))
|
||||
for row in conn.execute(stmt):
|
||||
ident = str(row[0])
|
||||
entry = result.get(ident)
|
||||
if entry is not None:
|
||||
entry["param"] = row[1]
|
||||
entry["sn"] = ident
|
||||
|
||||
def _fill_id() -> None:
|
||||
if not id_items:
|
||||
@@ -119,14 +133,15 @@ def fetch_params_by_ids(
|
||||
logger.warning(
|
||||
"未配置 database.id_field,--id 将退化为按总排号列 [%s] 查询", sn_col
|
||||
)
|
||||
stmt = select(sn_c, param_c).where(sn_c.in_(id_items))
|
||||
with engine.connect() as conn:
|
||||
for row in conn.execute(stmt):
|
||||
ident = str(row[0])
|
||||
entry = result.get(ident)
|
||||
if entry is not None:
|
||||
entry["param"] = row[1]
|
||||
entry["sn"] = ident
|
||||
for chunk in _chunked(id_items):
|
||||
stmt = select(sn_c, param_c).where(sn_c.in_(chunk))
|
||||
for row in conn.execute(stmt):
|
||||
ident = str(row[0])
|
||||
entry = result.get(ident)
|
||||
if entry is not None:
|
||||
entry["param"] = row[1]
|
||||
entry["sn"] = ident
|
||||
return
|
||||
|
||||
id_c = src.c[id_field]
|
||||
@@ -144,17 +159,18 @@ def fetch_params_by_ids(
|
||||
if not id_ints:
|
||||
return
|
||||
|
||||
stmt = select(id_c, sn_c, param_c).where(id_c.in_(id_ints))
|
||||
with engine.connect() as conn:
|
||||
for row in conn.execute(stmt):
|
||||
idv = row[0]
|
||||
orig = id_int_map.get(idv)
|
||||
if orig is None:
|
||||
continue
|
||||
entry = result.get(orig)
|
||||
if entry is not None:
|
||||
entry["param"] = row[2]
|
||||
entry["sn"] = row[1]
|
||||
for chunk in _chunked(id_ints):
|
||||
stmt = select(id_c, sn_c, param_c).where(id_c.in_(chunk))
|
||||
for row in conn.execute(stmt):
|
||||
idv = row[0]
|
||||
orig = id_int_map.get(idv)
|
||||
if orig is None:
|
||||
continue
|
||||
entry = result.get(orig)
|
||||
if entry is not None:
|
||||
entry["param"] = row[2]
|
||||
entry["sn"] = row[1]
|
||||
|
||||
try:
|
||||
_fill_sn()
|
||||
@@ -227,6 +243,26 @@ def fetch_all_ids(
|
||||
raise DatabaseError(f"查询总排号失败: {e}") from e
|
||||
|
||||
|
||||
def fetch_existing_attachment_sns(db_cfg: dict[str, Any]) -> set[str]:
|
||||
"""读取 Common.Attachment 中已存在的全部 SN(distinct)。
|
||||
|
||||
用于"追加"模式做差集:源表总排号集合 减去 这里返回的 SN 集合,
|
||||
得到"源表有、但目标表还没写过"的总排号,只对这部分分类写入。
|
||||
|
||||
返回 set[str](空表返回空集合);SN 统一转成 str,便于与源表总排号
|
||||
(同样是 str)直接做集合比较。NULL 值会被过滤掉。
|
||||
"""
|
||||
engine = get_engine(db_cfg)
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
stmt = select(Attachment.SN).distinct()
|
||||
return {
|
||||
str(row[0]) for row in conn.execute(stmt) if row[0] is not None
|
||||
}
|
||||
except SQLAlchemyError as e:
|
||||
raise DatabaseError(f"读取已有附件 SN 失败: {e}") from e
|
||||
|
||||
|
||||
def upsert_attachments(
|
||||
db_cfg: dict[str, Any],
|
||||
rows: list[tuple[str, str, str]],
|
||||
@@ -237,6 +273,9 @@ def upsert_attachments(
|
||||
对出现的每个 SN 先 DELETE 其旧行,再批量 INSERT——保证重跑总是反映
|
||||
最新分类结果,不会因唯一约束(复合主键 (SN,大类,小类))冲突而失败。
|
||||
|
||||
DELETE 的 IN 列表按 _PARAM_CHUNK 分片执行,避免 SN 过多时单条语句超过
|
||||
SQL Server 的 2100 绑定参数上限;整个"删 + 插"在同一事务内完成,语义不变。
|
||||
|
||||
返回 (deleted_rows, inserted_rows) 计数。
|
||||
"""
|
||||
if not rows:
|
||||
@@ -247,10 +286,14 @@ def upsert_attachments(
|
||||
|
||||
try:
|
||||
with Session(engine) as session:
|
||||
del_res = session.execute(
|
||||
delete(Attachment).where(Attachment.SN.in_(distinct_sn))
|
||||
)
|
||||
deleted = del_res.rowcount if del_res.rowcount is not None else 0
|
||||
# 分片 DELETE:单条 IN (...) 最多 _PARAM_CHUNK 个参数,避免触发 2100 上限
|
||||
deleted = 0
|
||||
for chunk in _chunked(distinct_sn):
|
||||
del_res = session.execute(
|
||||
delete(Attachment).where(Attachment.SN.in_(chunk))
|
||||
)
|
||||
if del_res.rowcount:
|
||||
deleted += del_res.rowcount
|
||||
session.execute(
|
||||
insert(Attachment),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user