feat(write_attachments): add --append mode, fix batch timing and SQL Server param limit
- Add --append flag: diff source-table 总排号 against existing Common.Attachment SN and classify/write only the missing ones. --limit caps the per-run append count, --order sets the direction. Backed by new fetch_existing_attachment_sns() in db.py. - Fix batch elapsed-time accounting: summarize_results() summed each task's per-item elapsed_ms, which overcounts under ThreadPoolExecutor concurrency (cumulative work time, not real wall-clock — 100 tasks on 8 workers reported ~5x the actual runtime). Callers now time classify_batch() via perf_counter and pass wall_clock_ms; both the write_attachments [汇总] line and main.py --summary report wall-clock separately from the cumulative sum. - Format durations >=1s in seconds (88851.4 ms -> 88.85 s) in the human-readable [汇总] line; structured JSON --summary fields stay in ms. - Chunk all IN (...) lists to 2000 items to respect SQL Server's 2100 bind-parameter hard limit (previously --append --limit 5000 failed at the fetch step with "COUNT 字段不正确"). Applied to fetch_params_by_ids (sn / id paths) and to the DELETE inside upsert_attachments. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,8 @@
|
||||
python write_attachments.py --limit 10 --order desc # 按真实 ID 降序取前 10 个
|
||||
python write_attachments.py --range 800,805 # 真实 ID 在 [800,805] 闭区间内
|
||||
python write_attachments.py --range 800,805 --order desc --limit 3 # 组合:区间内降序取前3
|
||||
python write_attachments.py --append --limit 10 # 追加模式:仅补写源表中还没写过的前 10 个总排号
|
||||
python write_attachments.py --append --order desc --limit 10 # 追加模式:从最大 ID 端开始补 10 个
|
||||
python write_attachments.py --ids-file ids.txt # 指定总排号(文件,每行一个)
|
||||
python write_attachments.py --sn 26B742 # 单个总排号写入(键类型 sn)
|
||||
python write_attachments.py --sn 26B742,26B743 # 多个总排号(逗号分隔)写入
|
||||
@@ -35,6 +37,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||||
@@ -49,12 +52,28 @@ from db import ( # noqa: E402
|
||||
SENTINEL_MINOR,
|
||||
NO_MINOR,
|
||||
fetch_all_ids,
|
||||
fetch_existing_attachment_sns,
|
||||
get_engine,
|
||||
init_schema,
|
||||
upsert_attachments,
|
||||
)
|
||||
|
||||
|
||||
def _format_duration(ms: float | None) -> str:
|
||||
"""把毫秒耗时格式化为人类友好的时长字符串。
|
||||
|
||||
超过 1 秒(>=1000ms)按秒输出(保留 2 位小数),否则按毫秒(保留 1 位小数)。
|
||||
例:88851.4 -> "88.85 s",458.9 -> "458.9 ms"。None / 异常值回退为 "0 ms"。
|
||||
"""
|
||||
try:
|
||||
v = float(ms) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return "0 ms"
|
||||
if v >= 1000:
|
||||
return f"{v / 1000:.2f} s"
|
||||
return f"{v:.1f} ms"
|
||||
|
||||
|
||||
def convert_results_to_rows(results: list[dict], mode: str) -> tuple[list[tuple[str, str, str]], list[str]]:
|
||||
"""将 classify_batch 的结果列表转成可写入 Common.Attachment 的行。
|
||||
|
||||
@@ -146,6 +165,13 @@ def main() -> None:
|
||||
"--ids-file", type=str, default=None,
|
||||
help="包含总排号的文本文件路径,每行一个总排号(键类型 sn);可单独使用或与 --id/--sn 合并",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--append", action="store_true",
|
||||
help="追加模式:对比源表全部总排号与 Common.Attachment 已有 SN,仅对"
|
||||
"「源表存在、Attachment 尚未写入」的总排号分类写入。--limit 此时是"
|
||||
"「本次最多追加多少个 SN」的上限,--order 决定从哪一端(升/降序)补起;"
|
||||
"与 --id/--sn/--ids-file 互斥",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode", type=str, default=None, choices=["coarse", "fine"],
|
||||
help="分类粒度,不指定则用配置文件 business.default_mode",
|
||||
@@ -194,6 +220,9 @@ def main() -> None:
|
||||
|
||||
# 1) 取总排号列表
|
||||
if args.id or args.sn or args.ids_file:
|
||||
if args.append:
|
||||
print("[错误] --append 不能与 --id/--sn/--ids-file 同时使用", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
ids = _parse_ids(args)
|
||||
if not ids:
|
||||
print("[错误] --id / --sn / --ids-file 未解析到任何有效的标识符", file=sys.stderr)
|
||||
@@ -201,6 +230,22 @@ def main() -> None:
|
||||
n_id = sum(1 for _, k in ids if k == "id")
|
||||
n_sn = sum(1 for _, k in ids if k == "sn")
|
||||
logger.info("从 --id/--sn/--ids-file 读取 %d 个标识符(id=%d, sn=%d)", len(ids), n_id, n_sn)
|
||||
elif args.append:
|
||||
# 追加模式:源表全部总排号 − Common.Attachment 已有 SN = 待补写集合。
|
||||
# 全表拉取源表总排号(按真实 ID 排序),再做差集;--order 控制补写方向,
|
||||
# --limit 此时是"本次最多追加多少个 SN"的上限。
|
||||
all_sns = fetch_all_ids(cfg["database"], order=args.order)
|
||||
existing = fetch_existing_attachment_sns(cfg["database"])
|
||||
missing = [s for s in all_sns if s not in existing]
|
||||
print(
|
||||
f"[追加] 源表 {len(all_sns)} 个总排号,Common.Attachment 已有 "
|
||||
f"{len(existing)} 个,待追加 {len(missing)} 个(order={args.order})"
|
||||
)
|
||||
if args.limit is not None:
|
||||
missing = missing[: args.limit]
|
||||
print(f"[追加] 应用上限 --limit {args.limit},本次将处理 {len(missing)} 个")
|
||||
ids = [(s, "sn") for s in missing]
|
||||
logger.info("追加模式:待处理 %d 个总排号", len(ids))
|
||||
else:
|
||||
# 全表扫描路径:--range/--order/--limit 仅在此生效
|
||||
if args.range:
|
||||
@@ -233,6 +278,10 @@ def main() -> None:
|
||||
|
||||
# 2) 分类
|
||||
logger.info("开始分类 (mode=%s, enable_other=%s)", mode, enable_other)
|
||||
# 整批分类的真实墙钟:在并发 classify_batch 外层计时(perf_counter)。
|
||||
# 注意 summarize_results 里累加的 total_elapsed_ms 是各任务自身耗时的总和
|
||||
# (并发下的"累计工作量",会明显大于这里测得的墙钟),二者之比 ≈ 并发增益。
|
||||
t0 = time.perf_counter()
|
||||
results = classify_batch(
|
||||
db_cfg=cfg["database"],
|
||||
llm_cfg=cfg["llm"],
|
||||
@@ -242,6 +291,7 @@ def main() -> None:
|
||||
log_dir=log_dir,
|
||||
enable_other=enable_other,
|
||||
)
|
||||
wall_ms = (time.perf_counter() - t0) * 1000
|
||||
|
||||
# 3) 转换
|
||||
rows, skipped = convert_results_to_rows(results, mode)
|
||||
@@ -261,12 +311,14 @@ def main() -> None:
|
||||
f"跳过(无附件判定=null) {len(skipped)} 个"
|
||||
)
|
||||
|
||||
# 运行汇总:总耗时/token 总量/缓存命中率,从每条结果的 meta 累加。
|
||||
# 运行汇总:墙钟耗时 + 各任务累计工作量 + token 总量/缓存命中率。
|
||||
# 走 stdout 与上面的 [统计] 一致;不属于落库动作本身,仅作诊断输出。
|
||||
summary = summarize_results(results)
|
||||
summary = summarize_results(results, wall_clock_ms=wall_ms)
|
||||
print(
|
||||
f"[汇总] 总耗时 {summary['total_elapsed_ms']} ms, "
|
||||
f"LLM 调用 {summary['llm_calls']} 次, "
|
||||
f"[汇总] 总耗时 {_format_duration(summary['wall_clock_ms'])}, "
|
||||
f"LLM 累计 {_format_duration(summary['total_elapsed_ms'])} "
|
||||
f"(并发 {cfg['business']['max_workers']}), "
|
||||
f"调用 {summary['llm_calls']} 次, "
|
||||
f"token: prompt={summary['total_prompt_tokens']} "
|
||||
f"completion={summary['total_completion_tokens']} "
|
||||
f"total={summary['total_tokens']}, "
|
||||
|
||||
Reference in New Issue
Block a user