feat: 附件分类结果写入层 + 运行统计 + --id 指定总排号
- db.py: 新增 upsert_attachments/fetch_all_ids,实现 Common.Attachment 幂等写入 (先删受影响 NS 旧行再批量插入,依赖唯一索引,可安全重跑) - write_attachments.py: 写入入口,支持 --limit/--ids-file/--id(单个或逗号分隔多个) /--mode/--dry-run/--enable-other,运行结束打印 token 与缓存命中率汇总 - llm_client.py: LLMCallResult 捕获 usage/elapsed_ms/model/attempt - classifier.py: classify_batch 结果透传 meta(耗时分两种、上下文 token、缓存命中率), 新增 summarize_results 聚合批统计 - main.py: 新增 --summary 把批汇总打到 stderr,stdout 保持干净 JSON Lines - order_logger.py: 每次 LLM 尝试补充 [调用统计] 段,便于排查耗时与缓存效果 - README.md: 对齐上述接口与统计说明 Co-Authored-By: WorkBuddy <workbuddy@tencent.com>
This commit is contained in:
86
db.py
86
db.py
@@ -86,3 +86,89 @@ def fetch_params_by_ids(
|
||||
def fetch_param_by_id(db_cfg: dict[str, Any], zong_pai_hao: str) -> str | None:
|
||||
"""单个总排号查询的便捷封装。"""
|
||||
return fetch_params_by_ids(db_cfg, [zong_pai_hao]).get(zong_pai_hao)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 附件分类落库(目标表 Common.Attachment,与源表 productionContractData 分属不同 schema)
|
||||
# ---------------------------------------------------------------------------
|
||||
# 落库约定(与项目记忆 MEMORY.md 中写入约定一致,请勿在此处用字面量以外的值):
|
||||
# - 无附件(has_attachment=false):写哨兵行 (NS, '无附件', '无')
|
||||
# - 无小类的行(coarse 真实附件 / 哨兵行):MinorCategory 统一填 '无'
|
||||
ATTACHMENT_SCHEMA = "Common"
|
||||
ATTACHMENT_TABLE = "Attachment"
|
||||
|
||||
SENTINEL_MAJOR = "无附件"
|
||||
SENTINEL_MINOR = "无"
|
||||
NO_MINOR = "无"
|
||||
|
||||
|
||||
def fetch_all_ids(db_cfg: dict[str, Any], limit: int | None = None) -> list[str]:
|
||||
"""拉取源表全部总排号(去重、排除 NULL),用于全量分类写入。
|
||||
|
||||
limit 用于测试时只取前 N 个(按总排号排序)。返回的总排号列表可直接
|
||||
喂给 classify_batch。
|
||||
"""
|
||||
schema = db_cfg["schema"]
|
||||
table = db_cfg["table"]
|
||||
id_col = db_cfg["id_column"]
|
||||
qualified = f"[{schema}].[{table}]"
|
||||
sql = f"SELECT DISTINCT [{id_col}] FROM {qualified} WHERE [{id_col}] IS NOT NULL"
|
||||
params: list[Any] = []
|
||||
if limit is not None:
|
||||
sql += f" ORDER BY [{id_col}] OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY"
|
||||
params.append(int(limit))
|
||||
conn_str = _build_conn_str(db_cfg)
|
||||
try:
|
||||
with pyodbc.connect(conn_str, timeout=db_cfg["connect_timeout"]) as conn:
|
||||
conn.timeout = db_cfg["query_timeout"]
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(sql, params)
|
||||
return [row[0] for row in cursor.fetchall()]
|
||||
except pyodbc.Error as e:
|
||||
raise DatabaseError(f"查询总排号失败: {e}") from e
|
||||
|
||||
|
||||
def upsert_attachments(
|
||||
db_cfg: dict[str, Any],
|
||||
rows: list[tuple[str, str, str]],
|
||||
) -> tuple[int, int]:
|
||||
"""幂等写入 Common.Attachment。
|
||||
|
||||
rows: 本次要写入的 (NS, MajorCategory, MinorCategory) 列表。
|
||||
对出现的每个 NS 先 DELETE 其旧行,再批量 INSERT——保证重跑总是反映
|
||||
最新分类结果,不会因唯一索引 (NS, 大类, 小类) 冲突而失败。
|
||||
|
||||
返回 (deleted_rows, inserted_rows) 计数。
|
||||
"""
|
||||
if not rows:
|
||||
return (0, 0)
|
||||
|
||||
qualified = f"[{ATTACHMENT_SCHEMA}].[{ATTACHMENT_TABLE}]"
|
||||
distinct_ns = sorted({r[0] for r in rows})
|
||||
|
||||
conn_str = _build_conn_str(db_cfg)
|
||||
try:
|
||||
with pyodbc.connect(conn_str, timeout=db_cfg["connect_timeout"]) as conn:
|
||||
conn.timeout = db_cfg["query_timeout"]
|
||||
cursor = conn.cursor()
|
||||
# 1) 删除本批所有 NS 的旧行
|
||||
del_ph = ",".join("?" for _ in distinct_ns)
|
||||
cursor.execute(
|
||||
f"DELETE FROM {qualified} WHERE [NS] IN ({del_ph})", distinct_ns
|
||||
)
|
||||
deleted = cursor.rowcount
|
||||
# 2) 插入新行
|
||||
cursor.executemany(
|
||||
f"INSERT INTO {qualified} ([NS], [MajorCategory], [MinorCategory]) "
|
||||
f"VALUES (?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
inserted = len(rows)
|
||||
conn.commit()
|
||||
logger.info(
|
||||
"upsert_attachments: 删除 %d 行, 插入 %d 行, 涉及 %d 个 NS",
|
||||
deleted, inserted, len(distinct_ns),
|
||||
)
|
||||
return (deleted, inserted)
|
||||
except pyodbc.Error as e:
|
||||
raise DatabaseError(f"写入附件表失败: {e}") from e
|
||||
|
||||
Reference in New Issue
Block a user