feat: 附件分类结果写入层 + 运行统计 + --id 指定总排号
- db.py: 新增 upsert_attachments/fetch_all_ids，实现 Common.Attachment 幂等写入 （先删受影响 NS 旧行再批量插入，依赖唯一索引，可安全重跑） - write_attachments.py: 写入入口，支持 --limit/--ids-file/--id（单个或逗号分隔多个） /--mode/--dry-run/--enable-other，运行结束打印 token 与缓存命中率汇总 - llm_client.py: LLMCallResult 捕获 usage/elapsed_ms/model/attempt - classifier.py: classify_batch 结果透传 meta（耗时分两种、上下文 token、缓存命中率）， 新增 summarize_results 聚合批统计 - main.py: 新增 --summary 把批汇总打到 stderr，stdout 保持干净 JSON Lines - order_logger.py: 每次 LLM 尝试补充 [调用统计] 段，便于排查耗时与缓存效果 - README.md: 对齐上述接口与统计说明 Co-Authored-By: WorkBuddy <workbuddy@tencent.com>
This commit is contained in:
Misaka_Company
228
write_attachments.py
Normal file
228
write_attachments.py
Normal file
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""基于源表总排号，调用 classify_batch 分类，将结果按约定写入 Common.Attachment。
|
||||
|
||||
落库约定（详见项目记忆 MEMORY.md，与 db.py 中的常量保持一致）：
|
||||
- has_attachment=true (status=ok):
|
||||
每个 type 一行。fine 模式 type 形如 "大类:细分"，按首个冒号拆成 (大类, 小类)；
|
||||
coarse 模式 type 是大类，小类统一填 '无'。
|
||||
- has_attachment=false (无附件): 写哨兵行 (NS, '无附件', '无')。
|
||||
- has_attachment=null (not_found / llm_error / db_error): 不写（属"无法确定/失败"）。
|
||||
|
||||
用法:
|
||||
python write_attachments.py # 全量写入
|
||||
python write_attachments.py --limit 10 # 仅前 10 个总排号（测试）
|
||||
python write_attachments.py --ids-file ids.txt # 指定总排号（文件，每行一个）
|
||||
python write_attachments.py --id 26B742 # 单个总排号写入
|
||||
python write_attachments.py --id 26B742,26B743 # 多个总排号（逗号分隔）写入
|
||||
python write_attachments.py --mode coarse # 粗分类写入
|
||||
python write_attachments.py --dry-run # 只打印将写入的行，不落库
|
||||
python write_attachments.py --config other.yaml
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||||
|
||||
from classifier import classify_batch, summarize_results # noqa: E402
|
||||
from config_loader import ConfigError, load_config # noqa: E402
|
||||
from db import ( # noqa: E402
|
||||
ATTACHMENT_SCHEMA,
|
||||
ATTACHMENT_TABLE,
|
||||
DatabaseError,
|
||||
SENTINEL_MAJOR,
|
||||
SENTINEL_MINOR,
|
||||
NO_MINOR,
|
||||
fetch_all_ids,
|
||||
upsert_attachments,
|
||||
)
|
||||
|
||||
|
||||
def convert_results_to_rows(results: list[dict], mode: str) -> tuple[list[tuple[str, str, str]], list[str]]:
|
||||
"""将 classify_batch 的结果列表转成可写入 Common.Attachment 的行。
|
||||
|
||||
返回 (rows, skipped)：
|
||||
rows - list[(NS, MajorCategory, MinorCategory)]
|
||||
skipped- list[总排号]，has_attachment 为 None（not_found/失败）未写入的
|
||||
"""
|
||||
rows: list[tuple[str, str, str]] = []
|
||||
skipped: list[str] = []
|
||||
for r in results:
|
||||
zph = r["zong_pai_hao"]
|
||||
has = r["has_attachment"]
|
||||
if has is None:
|
||||
skipped.append(zph)
|
||||
continue
|
||||
if has is True:
|
||||
for t in r["types"]:
|
||||
t = t.strip()
|
||||
if not t:
|
||||
continue
|
||||
if mode == "fine" and ":" in t:
|
||||
major, minor = t.split(":", 1)
|
||||
major = major.strip()
|
||||
minor = minor.strip() or NO_MINOR
|
||||
else:
|
||||
major = t
|
||||
minor = NO_MINOR
|
||||
rows.append((zph, major, minor))
|
||||
else: # has is False -> 无附件
|
||||
rows.append((zph, SENTINEL_MAJOR, SENTINEL_MINOR))
|
||||
return rows, skipped
|
||||
|
||||
|
||||
def _parse_ids(args: argparse.Namespace) -> list[str]:
|
||||
"""从 --id（逗号分隔，单个或多个）或 --ids-file 解析出总排号列表，去除空白项。
|
||||
|
||||
两个来源可同时提供，合并后返回（顺序：先 --id，后 --ids-file）。
|
||||
--ids-file 文件不存在时直接报错退出。
|
||||
"""
|
||||
ids: list[str] = []
|
||||
if args.id:
|
||||
ids.extend(s.strip() for s in args.id.split(",") if s.strip())
|
||||
if args.ids_file:
|
||||
p = Path(args.ids_file)
|
||||
if not p.exists():
|
||||
print(f"[错误] 总排号文件不存在: {p.resolve()}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
ids.extend(line.strip() for line in f if line.strip())
|
||||
return ids
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="将订单附件分类结果写入 Common.Attachment",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
parser.add_argument("--config", type=str, default="config.yaml")
|
||||
parser.add_argument(
|
||||
"--id", type=str, default=None,
|
||||
help="总排号，单个或逗号分隔的多个，例如: 26B742 或 26B742,26B743；"
|
||||
"提供后直接按指定总排号写入，不再全表扫描（--limit 此时无效）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit", type=int, default=None,
|
||||
help="仅处理前 N 个总排号（按总排号排序），测试用；与 --id/--ids-file 互斥",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ids-file", type=str, default=None,
|
||||
help="包含总排号的文本文件路径，每行一个总排号；可单独使用或与 --id 合并",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode", type=str, default=None, choices=["coarse", "fine"],
|
||||
help="分类粒度，不指定则用配置文件 business.default_mode",
|
||||
)
|
||||
parser.add_argument("--log-dir", type=str, default=None)
|
||||
parser.add_argument("--enable-other", action="store_true")
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="只打印将要写入/跳过的统计，不连接目标表写入",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
cfg = load_config(args.config)
|
||||
except ConfigError as e:
|
||||
print(f"[配置错误] {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, cfg["business"]["log_level"], logging.INFO),
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
logger = logging.getLogger("write_attachments")
|
||||
|
||||
mode = args.mode or cfg["business"]["default_mode"]
|
||||
log_dir = args.log_dir or cfg["business"]["log_dir"]
|
||||
# enable_other_category 是可选配置项，缺省按"关闭"；--enable-other 只能从关到开
|
||||
enable_other = bool(args.enable_other) or bool(
|
||||
cfg["business"].get("enable_other_category", False)
|
||||
)
|
||||
|
||||
# 1) 取总排号列表
|
||||
if args.id or args.ids_file:
|
||||
ids = _parse_ids(args)
|
||||
if not ids:
|
||||
print("[错误] --id / --ids-file 未解析到任何有效的总排号", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
logger.info("从 --id / --ids-file 读取 %d 个总排号", len(ids))
|
||||
else:
|
||||
ids = fetch_all_ids(cfg["database"], limit=args.limit)
|
||||
logger.info(
|
||||
"源表读取 %d 个总排号%s", len(ids),
|
||||
f"（limit={args.limit}）" if args.limit else "",
|
||||
)
|
||||
|
||||
if not ids:
|
||||
print("[提示] 没有可处理的总排号，退出")
|
||||
return
|
||||
|
||||
# 2) 分类
|
||||
logger.info("开始分类 (mode=%s, enable_other=%s)", mode, enable_other)
|
||||
results = classify_batch(
|
||||
db_cfg=cfg["database"],
|
||||
llm_cfg=cfg["llm"],
|
||||
zong_pai_hao_list=ids,
|
||||
max_workers=cfg["business"]["max_workers"],
|
||||
mode=mode,
|
||||
log_dir=log_dir,
|
||||
enable_other=enable_other,
|
||||
)
|
||||
|
||||
# 3) 转换
|
||||
rows, skipped = convert_results_to_rows(results, mode)
|
||||
|
||||
# 统计各状态
|
||||
n_ok = sum(1 for r in results if r["status"] == "ok")
|
||||
n_empty = sum(1 for r in results if r["status"] == "empty_param")
|
||||
n_notfound = sum(1 for r in results if r["status"] == "not_found")
|
||||
n_err = sum(1 for r in results if r["status"].endswith("error"))
|
||||
|
||||
print(
|
||||
f"[统计] 输入 {len(results)} 个总排号: "
|
||||
f"ok={n_ok} empty_param={n_empty} not_found={n_notfound} error={n_err}"
|
||||
)
|
||||
print(
|
||||
f"[统计] 将写入 {len(rows)} 行, "
|
||||
f"跳过(无附件判定=null) {len(skipped)} 个"
|
||||
)
|
||||
|
||||
# 运行汇总：总耗时/token 总量/缓存命中率，从每条结果的 meta 累加。
|
||||
# 走 stdout 与上面的 [统计] 一致；不属于落库动作本身，仅作诊断输出。
|
||||
summary = summarize_results(results)
|
||||
print(
|
||||
f"[汇总] 总耗时 {summary['total_elapsed_ms']} ms, "
|
||||
f"LLM 调用 {summary['llm_calls']} 次, "
|
||||
f"token: prompt={summary['total_prompt_tokens']} "
|
||||
f"completion={summary['total_completion_tokens']} "
|
||||
f"total={summary['total_tokens']}, "
|
||||
f"缓存命中率 {summary['avg_cache_hit_rate']} "
|
||||
f"(hit={summary['total_cache_hit_tokens']} "
|
||||
f"miss={summary['total_cache_miss_tokens']})"
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
print("[dry-run] 不落库。示例行（前 10）:")
|
||||
for row in rows[:10]:
|
||||
print(" ", row)
|
||||
return
|
||||
|
||||
# 4) 写入
|
||||
target = f"{ATTACHMENT_SCHEMA}.{ATTACHMENT_TABLE}"
|
||||
try:
|
||||
deleted, inserted = upsert_attachments(cfg["database"], rows)
|
||||
print(f"[完成] 已写入 {target}: 删除旧行 {deleted}, 插入新行 {inserted}")
|
||||
except DatabaseError as e:
|
||||
print(f"[写入失败] {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user