feat(write_attachments): add --append mode, fix batch timing and SQL Server param limit
- Add --append flag: diff source-table 总排号 against existing Common.Attachment SN and classify/write only the missing ones. --limit caps the per-run append count, --order sets the direction. Backed by new fetch_existing_attachment_sns() in db.py. - Fix batch elapsed-time accounting: summarize_results() summed each task's per-item elapsed_ms, which overcounts under ThreadPoolExecutor concurrency (cumulative work time, not real wall-clock — 100 tasks on 8 workers reported ~5x the actual runtime). Callers now time classify_batch() via perf_counter and pass wall_clock_ms; both the write_attachments [汇总] line and main.py --summary report wall-clock separately from the cumulative sum. - Format durations >=1s in seconds (88851.4 ms -> 88.85 s) in the human-readable [汇总] line; structured JSON --summary fields stay in ms. - Chunk all IN (...) lists to 2000 items to respect SQL Server's 2100 bind-parameter hard limit (previously --append --limit 5000 failed at the fetch step with "COUNT 字段不正确"). Applied to fetch_params_by_ids (sn / id paths) and to the DELETE inside upsert_attachments. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
8
main.py
8
main.py
@@ -62,6 +62,7 @@ import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||||
@@ -166,6 +167,10 @@ def main() -> None:
|
||||
# 只能从"关"打开成"开",不支持反过来用命令行强制关闭配置文件里已打开的设置。
|
||||
enable_other = bool(args.enable_other) or bool(cfg["business"].get("enable_other_category", False))
|
||||
|
||||
# 整批分类的真实墙钟:在并发 classify_batch 外层计时(perf_counter)。
|
||||
# summarize_results 累加的 total_elapsed_ms 是各任务自身耗时的总和(并发下的
|
||||
# 累计工作量,会明显大于墙钟),二者之比 ≈ 并发增益;wall_clock_ms 才是真实墙钟。
|
||||
t0 = time.perf_counter()
|
||||
results = classify_batch(
|
||||
db_cfg=cfg["database"],
|
||||
llm_cfg=cfg["llm"],
|
||||
@@ -175,6 +180,7 @@ def main() -> None:
|
||||
log_dir=log_dir,
|
||||
enable_other=enable_other,
|
||||
)
|
||||
wall_ms = (time.perf_counter() - t0) * 1000
|
||||
|
||||
if args.pretty:
|
||||
print(json.dumps(results, ensure_ascii=False, indent=2))
|
||||
@@ -183,7 +189,7 @@ def main() -> None:
|
||||
print(json.dumps(r, ensure_ascii=False))
|
||||
|
||||
if args.summary:
|
||||
summary = summarize_results(results)
|
||||
summary = summarize_results(results, wall_clock_ms=wall_ms)
|
||||
print("--- 运行汇总 ---", file=sys.stderr)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2), file=sys.stderr)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user