622f348cf1
- db.py: 新增 upsert_attachments/fetch_all_ids，实现 Common.Attachment 幂等写入 （先删受影响 NS 旧行再批量插入，依赖唯一索引，可安全重跑） - write_attachments.py: 写入入口，支持 --limit/--ids-file/--id（单个或逗号分隔多个） /--mode/--dry-run/--enable-other，运行结束打印 token 与缓存命中率汇总 - llm_client.py: LLMCallResult 捕获 usage/elapsed_ms/model/attempt - classifier.py: classify_batch 结果透传 meta（耗时分两种、上下文 token、缓存命中率）， 新增 summarize_results 聚合批统计 - main.py: 新增 --summary 把批汇总打到 stderr，stdout 保持干净 JSON Lines - order_logger.py: 每次 LLM 尝试补充 [调用统计] 段，便于排查耗时与缓存效果 - README.md: 对齐上述接口与统计说明 Co-Authored-By: WorkBuddy <workbuddy@tencent.com>
229 lines
8.6 KiB
Python
229 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""基于源表总排号，调用 classify_batch 分类，将结果按约定写入 Common.Attachment。
|
||
|
||
落库约定（详见项目记忆 MEMORY.md，与 db.py 中的常量保持一致）：
|
||
- has_attachment=true (status=ok):
|
||
每个 type 一行。fine 模式 type 形如 "大类:细分"，按首个冒号拆成 (大类, 小类)；
|
||
coarse 模式 type 是大类，小类统一填 '无'。
|
||
- has_attachment=false (无附件): 写哨兵行 (NS, '无附件', '无')。
|
||
- has_attachment=null (not_found / llm_error / db_error): 不写（属"无法确定/失败"）。
|
||
|
||
用法:
|
||
python write_attachments.py # 全量写入
|
||
python write_attachments.py --limit 10 # 仅前 10 个总排号（测试）
|
||
python write_attachments.py --ids-file ids.txt # 指定总排号（文件，每行一个）
|
||
python write_attachments.py --id 26B742 # 单个总排号写入
|
||
python write_attachments.py --id 26B742,26B743 # 多个总排号（逗号分隔）写入
|
||
python write_attachments.py --mode coarse # 粗分类写入
|
||
python write_attachments.py --dry-run # 只打印将写入的行，不落库
|
||
python write_attachments.py --config other.yaml
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import logging
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||
|
||
from classifier import classify_batch, summarize_results # noqa: E402
|
||
from config_loader import ConfigError, load_config # noqa: E402
|
||
from db import ( # noqa: E402
|
||
ATTACHMENT_SCHEMA,
|
||
ATTACHMENT_TABLE,
|
||
DatabaseError,
|
||
SENTINEL_MAJOR,
|
||
SENTINEL_MINOR,
|
||
NO_MINOR,
|
||
fetch_all_ids,
|
||
upsert_attachments,
|
||
)
|
||
|
||
|
||
def convert_results_to_rows(results: list[dict], mode: str) -> tuple[list[tuple[str, str, str]], list[str]]:
|
||
"""将 classify_batch 的结果列表转成可写入 Common.Attachment 的行。
|
||
|
||
返回 (rows, skipped)：
|
||
rows - list[(NS, MajorCategory, MinorCategory)]
|
||
skipped- list[总排号]，has_attachment 为 None（not_found/失败）未写入的
|
||
"""
|
||
rows: list[tuple[str, str, str]] = []
|
||
skipped: list[str] = []
|
||
for r in results:
|
||
zph = r["zong_pai_hao"]
|
||
has = r["has_attachment"]
|
||
if has is None:
|
||
skipped.append(zph)
|
||
continue
|
||
if has is True:
|
||
for t in r["types"]:
|
||
t = t.strip()
|
||
if not t:
|
||
continue
|
||
if mode == "fine" and ":" in t:
|
||
major, minor = t.split(":", 1)
|
||
major = major.strip()
|
||
minor = minor.strip() or NO_MINOR
|
||
else:
|
||
major = t
|
||
minor = NO_MINOR
|
||
rows.append((zph, major, minor))
|
||
else: # has is False -> 无附件
|
||
rows.append((zph, SENTINEL_MAJOR, SENTINEL_MINOR))
|
||
return rows, skipped
|
||
|
||
|
||
def _parse_ids(args: argparse.Namespace) -> list[str]:
|
||
"""从 --id（逗号分隔，单个或多个）或 --ids-file 解析出总排号列表，去除空白项。
|
||
|
||
两个来源可同时提供，合并后返回（顺序：先 --id，后 --ids-file）。
|
||
--ids-file 文件不存在时直接报错退出。
|
||
"""
|
||
ids: list[str] = []
|
||
if args.id:
|
||
ids.extend(s.strip() for s in args.id.split(",") if s.strip())
|
||
if args.ids_file:
|
||
p = Path(args.ids_file)
|
||
if not p.exists():
|
||
print(f"[错误] 总排号文件不存在: {p.resolve()}", file=sys.stderr)
|
||
sys.exit(1)
|
||
with p.open("r", encoding="utf-8") as f:
|
||
ids.extend(line.strip() for line in f if line.strip())
|
||
return ids
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(
|
||
description="将订单附件分类结果写入 Common.Attachment",
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
epilog=__doc__,
|
||
)
|
||
parser.add_argument("--config", type=str, default="config.yaml")
|
||
parser.add_argument(
|
||
"--id", type=str, default=None,
|
||
help="总排号，单个或逗号分隔的多个，例如: 26B742 或 26B742,26B743；"
|
||
"提供后直接按指定总排号写入，不再全表扫描（--limit 此时无效）",
|
||
)
|
||
parser.add_argument(
|
||
"--limit", type=int, default=None,
|
||
help="仅处理前 N 个总排号（按总排号排序），测试用；与 --id/--ids-file 互斥",
|
||
)
|
||
parser.add_argument(
|
||
"--ids-file", type=str, default=None,
|
||
help="包含总排号的文本文件路径，每行一个总排号；可单独使用或与 --id 合并",
|
||
)
|
||
parser.add_argument(
|
||
"--mode", type=str, default=None, choices=["coarse", "fine"],
|
||
help="分类粒度，不指定则用配置文件 business.default_mode",
|
||
)
|
||
parser.add_argument("--log-dir", type=str, default=None)
|
||
parser.add_argument("--enable-other", action="store_true")
|
||
parser.add_argument(
|
||
"--dry-run", action="store_true",
|
||
help="只打印将要写入/跳过的统计，不连接目标表写入",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
try:
|
||
cfg = load_config(args.config)
|
||
except ConfigError as e:
|
||
print(f"[配置错误] {e}", file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
logging.basicConfig(
|
||
level=getattr(logging, cfg["business"]["log_level"], logging.INFO),
|
||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||
stream=sys.stderr,
|
||
)
|
||
logger = logging.getLogger("write_attachments")
|
||
|
||
mode = args.mode or cfg["business"]["default_mode"]
|
||
log_dir = args.log_dir or cfg["business"]["log_dir"]
|
||
# enable_other_category 是可选配置项，缺省按"关闭"；--enable-other 只能从关到开
|
||
enable_other = bool(args.enable_other) or bool(
|
||
cfg["business"].get("enable_other_category", False)
|
||
)
|
||
|
||
# 1) 取总排号列表
|
||
if args.id or args.ids_file:
|
||
ids = _parse_ids(args)
|
||
if not ids:
|
||
print("[错误] --id / --ids-file 未解析到任何有效的总排号", file=sys.stderr)
|
||
sys.exit(1)
|
||
logger.info("从 --id / --ids-file 读取 %d 个总排号", len(ids))
|
||
else:
|
||
ids = fetch_all_ids(cfg["database"], limit=args.limit)
|
||
logger.info(
|
||
"源表读取 %d 个总排号%s", len(ids),
|
||
f"（limit={args.limit}）" if args.limit else "",
|
||
)
|
||
|
||
if not ids:
|
||
print("[提示] 没有可处理的总排号，退出")
|
||
return
|
||
|
||
# 2) 分类
|
||
logger.info("开始分类 (mode=%s, enable_other=%s)", mode, enable_other)
|
||
results = classify_batch(
|
||
db_cfg=cfg["database"],
|
||
llm_cfg=cfg["llm"],
|
||
zong_pai_hao_list=ids,
|
||
max_workers=cfg["business"]["max_workers"],
|
||
mode=mode,
|
||
log_dir=log_dir,
|
||
enable_other=enable_other,
|
||
)
|
||
|
||
# 3) 转换
|
||
rows, skipped = convert_results_to_rows(results, mode)
|
||
|
||
# 统计各状态
|
||
n_ok = sum(1 for r in results if r["status"] == "ok")
|
||
n_empty = sum(1 for r in results if r["status"] == "empty_param")
|
||
n_notfound = sum(1 for r in results if r["status"] == "not_found")
|
||
n_err = sum(1 for r in results if r["status"].endswith("error"))
|
||
|
||
print(
|
||
f"[统计] 输入 {len(results)} 个总排号: "
|
||
f"ok={n_ok} empty_param={n_empty} not_found={n_notfound} error={n_err}"
|
||
)
|
||
print(
|
||
f"[统计] 将写入 {len(rows)} 行, "
|
||
f"跳过(无附件判定=null) {len(skipped)} 个"
|
||
)
|
||
|
||
# 运行汇总：总耗时/token 总量/缓存命中率，从每条结果的 meta 累加。
|
||
# 走 stdout 与上面的 [统计] 一致；不属于落库动作本身，仅作诊断输出。
|
||
summary = summarize_results(results)
|
||
print(
|
||
f"[汇总] 总耗时 {summary['total_elapsed_ms']} ms, "
|
||
f"LLM 调用 {summary['llm_calls']} 次, "
|
||
f"token: prompt={summary['total_prompt_tokens']} "
|
||
f"completion={summary['total_completion_tokens']} "
|
||
f"total={summary['total_tokens']}, "
|
||
f"缓存命中率 {summary['avg_cache_hit_rate']} "
|
||
f"(hit={summary['total_cache_hit_tokens']} "
|
||
f"miss={summary['total_cache_miss_tokens']})"
|
||
)
|
||
|
||
if args.dry_run:
|
||
print("[dry-run] 不落库。示例行（前 10）:")
|
||
for row in rows[:10]:
|
||
print(" ", row)
|
||
return
|
||
|
||
# 4) 写入
|
||
target = f"{ATTACHMENT_SCHEMA}.{ATTACHMENT_TABLE}"
|
||
try:
|
||
deleted, inserted = upsert_attachments(cfg["database"], rows)
|
||
print(f"[完成] 已写入 {target}: 删除旧行 {deleted}, 插入新行 {inserted}")
|
||
except DatabaseError as e:
|
||
print(f"[写入失败] {e}", file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|