feat(db_compare): add PostgreSQL-based comparison engine with SF handling
Replace Excel-based undelivered comparison with DB queries for all four sites. The engine anchors on actual scan_time, reverse-lookups handover batches, and compares expected vs actual waybill-by-waybill. Shunxin SF waybills: use COUNT(*) instead of COUNT(DISTINCT piece_no) since SF piece numbers are random and not derivable from the waybill. Changes: - db_compare.py: new module with compare_site_date(), compare_site_batch(), write_result_excel(), and POST /compare API endpoint - runtime.py: switch _site_undelivered_handler from compare.write_site_file (Excel) to db_compare (DB); downloads succeed independently of comparison - server.py: add POST /compare endpoint with date validation - docs: implementation plan for Shunxin DB comparison Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
534
inbound_verify/db_compare.py
Normal file
534
inbound_verify/db_compare.py
Normal file
@@ -0,0 +1,534 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
db_compare.py — 基于 PostgreSQL 的应到未到差缺比对引擎。
|
||||
|
||||
与 compare.py（Excel 版）并行：本模块直接从 DB 查询数据进行比对，
|
||||
不依赖 downloads/ 下的 Excel 文件。
|
||||
|
||||
核心思路：以实到扫描日期为锚点 → 反推交接批次 → 展开批次全量比对。
|
||||
|
||||
每个站点只需提供配置（waybill 列名 / piece 列名 / 是否有 SF 特殊处理），
|
||||
核心比对逻辑完全通用。
|
||||
|
||||
顺心站点 SF 运单特殊处理：SF 运单的子单号（piece_no）为随机号码，不能用
|
||||
COUNT(DISTINCT piece_no) 去重计数，改为 COUNT(*) 行计数。
|
||||
|
||||
用法：
|
||||
from inbound_verify.db_compare import compare_site_date, SITE_COMPARE_CONFIG
|
||||
|
||||
result = compare_site_date("顺心", "2026-07-25")
|
||||
if result:
|
||||
print(result.stats)
|
||||
for row in result.rows:
|
||||
print(row)
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
|
||||
import psycopg
|
||||
import yaml
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side
|
||||
|
||||
from inbound_verify.paths import CONFIG_PATH, OUTPUT_DIR, DOWNLOAD_DIR
|
||||
from inbound_verify.domain import _site_cfg, ALL_REPORT_SITES
|
||||
|
||||
# ============================== 结果类型 ==============================
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareStats:
|
||||
"""单站点/单批次比对统计。"""
|
||||
|
||||
waybill_count: int = 0 # 应到运单数
|
||||
expected_pieces: int = 0 # 应到件数
|
||||
arrived_pieces: int = 0 # 实到件数
|
||||
undelivered_pieces: int = 0 # 未到件数
|
||||
undelivered_wb: int = 0 # 差缺运单数
|
||||
full_miss: int = 0 # 完全未到
|
||||
part_miss: int = 0 # 部分未到
|
||||
sf_wb_count: int = 0 # SF 运单数
|
||||
sf_undelivered: int = 0 # SF 差缺数
|
||||
|
||||
|
||||
@dataclass
|
||||
class UndeliveredRow:
|
||||
"""单条差缺明细。"""
|
||||
|
||||
handover_no: str = "" # 交接单号
|
||||
waybill_no: str = "" # 运单号
|
||||
total_pieces: int = 0 # 总件数（交接件数）
|
||||
arrived_pieces: int = 0 # 已到件数
|
||||
arrived_list: list = field(default_factory=list) # 已到单号列表
|
||||
is_sf: bool = False # 是否 SF 运单
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareResult:
|
||||
"""一次比对的完整结果。"""
|
||||
|
||||
site: str = ""
|
||||
date: str = ""
|
||||
batches: list = field(default_factory=list) # 涉及的交接批次
|
||||
stats: CompareStats = field(default_factory=CompareStats)
|
||||
rows: list = field(default_factory=list) # UndeliveredRow 列表
|
||||
|
||||
|
||||
# ============================== 站点比对配置 ==============================
|
||||
|
||||
|
||||
@dataclass
|
||||
class SiteCompareConfig:
|
||||
"""DB 比对的站点参数。"""
|
||||
|
||||
name: str # 站点名
|
||||
has_sf: bool = False # 是否需要区分 SF 运单
|
||||
|
||||
|
||||
# 四站点 DB 比对配置（百世不参与 4 站比对）
|
||||
SITE_COMPARE_CONFIG: dict[str, SiteCompareConfig] = {
|
||||
"顺心": SiteCompareConfig(name="顺心", has_sf=True),
|
||||
"中通": SiteCompareConfig(name="中通", has_sf=False),
|
||||
"韵达": SiteCompareConfig(name="韵达", has_sf=False),
|
||||
"安能": SiteCompareConfig(name="安能", has_sf=False),
|
||||
}
|
||||
|
||||
|
||||
# ============================== DB 连接 ==============================
|
||||
|
||||
|
||||
def _load_pg_config():
|
||||
"""从 config.yaml 读 postgres 段。与 store.py 共用同一配置源。"""
|
||||
if not os.path.exists(CONFIG_PATH):
|
||||
raise FileNotFoundError(
|
||||
f"未找到配置文件 {CONFIG_PATH}（请参考 config.example.yaml 创建 config.yaml）"
|
||||
)
|
||||
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
cfg = yaml.safe_load(f) or {}
|
||||
pg = cfg.get("postgres") or {}
|
||||
return {
|
||||
"host": pg.get("host", "127.0.0.1"),
|
||||
"port": int(pg.get("port", 5432)),
|
||||
"user": pg.get("user", "postgres"),
|
||||
"password": pg.get("password", ""),
|
||||
"dbname": pg.get("dbname", "CQHXDB"),
|
||||
"schema": pg.get("schema", "inbound_verify"),
|
||||
"connect_timeout_seconds": int(pg.get("connect_timeout_seconds", 5)),
|
||||
}
|
||||
|
||||
|
||||
def _connect():
|
||||
c = _load_pg_config()
|
||||
return psycopg.connect(
|
||||
host=c["host"],
|
||||
port=c["port"],
|
||||
dbname=c["dbname"],
|
||||
user=c["user"],
|
||||
password=c["password"],
|
||||
options=f"-c search_path={c['schema']} -c statement_timeout=30s",
|
||||
connect_timeout=c["connect_timeout_seconds"],
|
||||
)
|
||||
|
||||
|
||||
# ============================== 核心比对逻辑 ==============================
|
||||
|
||||
|
||||
def compare_site_date(site: str, target_date: str) -> CompareResult | None:
|
||||
"""对指定站点和日期执行 DB 差缺比对。
|
||||
|
||||
算法：
|
||||
1. 取 scan_time::date = target_date 的实到运单（锚点）
|
||||
2. 反推这些运单所属的交接批次（handover_no）
|
||||
3. 展开批次全量应到运单
|
||||
4. 查询批次全量实到扫描
|
||||
5. 逐运单比对差缺（SF/non-SF 分支处理）
|
||||
|
||||
Args:
|
||||
site: 站点名（"顺心"/"中通"/"韵达"/"安能"）
|
||||
target_date: 日期 "YYYY-MM-DD"
|
||||
|
||||
Returns:
|
||||
CompareResult 或 None（当天无实到数据时返回 None）
|
||||
"""
|
||||
cfg = SITE_COMPARE_CONFIG.get(site)
|
||||
if cfg is None:
|
||||
print(f"[db_compare] 不支持的站点: {site}")
|
||||
return None
|
||||
|
||||
try:
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
|
||||
# ── Step 1: 取实到锚点 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT DISTINCT waybill_no FROM actual_record
|
||||
WHERE site = %s AND scan_time::date = %s
|
||||
""",
|
||||
(site, target_date),
|
||||
)
|
||||
anchor_wbs = [r[0] for r in cur.fetchall()]
|
||||
if not anchor_wbs:
|
||||
print(f"[db_compare] {site} {target_date}: 当天无实到数据")
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
# ── Step 2: 反推交接批次 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT DISTINCT e.handover_no FROM expected_record e
|
||||
WHERE e.site = %s AND e.waybill_no = ANY(%s)
|
||||
""",
|
||||
(site, anchor_wbs),
|
||||
)
|
||||
batches = [r[0] for r in cur.fetchall()]
|
||||
|
||||
# ── Step 3: 展开批次全量应到 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, handover_no, handover_pieces
|
||||
FROM expected_record
|
||||
WHERE site = %s AND handover_no = ANY(%s)
|
||||
ORDER BY handover_no, waybill_no
|
||||
""",
|
||||
(site, batches),
|
||||
)
|
||||
exp_rows = cur.fetchall() # [(waybill_no, handover_no, handover_pieces), ...]
|
||||
|
||||
if not exp_rows:
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
all_wbs = [r[0] for r in exp_rows]
|
||||
|
||||
# ── Step 4: 取批次全量实到 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, piece_no FROM actual_record
|
||||
WHERE site = %s AND waybill_no = ANY(%s)
|
||||
ORDER BY waybill_no, piece_no
|
||||
""",
|
||||
(site, all_wbs),
|
||||
)
|
||||
act_rows = cur.fetchall() # [(waybill_no, piece_no), ...]
|
||||
|
||||
conn.close()
|
||||
|
||||
# ── Step 5: 逐运单比对 ──
|
||||
return _do_compare(site, target_date, batches, exp_rows, act_rows, cfg)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[db_compare] {site} {target_date} 比对异常: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def compare_site_batch(site: str, handover_no: str) -> CompareResult | None:
|
||||
"""按指定交接单号执行全批次比对（不依赖实到锚点）。
|
||||
|
||||
用于已知交接单号后精确比对某一批次。
|
||||
"""
|
||||
cfg = SITE_COMPARE_CONFIG.get(site)
|
||||
if cfg is None:
|
||||
print(f"[db_compare] 不支持的站点: {site}")
|
||||
return None
|
||||
|
||||
try:
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, handover_no, handover_pieces
|
||||
FROM expected_record
|
||||
WHERE site = %s AND handover_no = %s
|
||||
ORDER BY waybill_no
|
||||
""",
|
||||
(site, handover_no),
|
||||
)
|
||||
exp_rows = cur.fetchall()
|
||||
if not exp_rows:
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
all_wbs = [r[0] for r in exp_rows]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, piece_no FROM actual_record
|
||||
WHERE site = %s AND waybill_no = ANY(%s)
|
||||
ORDER BY waybill_no, piece_no
|
||||
""",
|
||||
(site, all_wbs),
|
||||
)
|
||||
act_rows = cur.fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
return _do_compare(
|
||||
site,
|
||||
f"batch:{handover_no}",
|
||||
[handover_no],
|
||||
exp_rows,
|
||||
act_rows,
|
||||
cfg,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[db_compare] {site} batch:{handover_no} 比对异常: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ============================== 比对核心 ==============================
|
||||
|
||||
|
||||
def _do_compare(
|
||||
site: str,
|
||||
label: str,
|
||||
batches: list[str],
|
||||
exp_rows: list[tuple], # [(waybill_no, handover_no, handover_pieces), ...]
|
||||
act_rows: list[tuple], # [(waybill_no, piece_no), ...]
|
||||
cfg: SiteCompareConfig,
|
||||
) -> CompareResult:
|
||||
"""执行逐运单比对，产出统计 + 差缺明细。
|
||||
|
||||
与 compare.py:process() 口径一致：
|
||||
- 应到件数 = handover_pieces（交接件数）
|
||||
- 实到件数 = SF ? COUNT(*) : COUNT(DISTINCT piece_no)
|
||||
- arrived_cnt >= handover_pieces → 足额到货，跳过
|
||||
"""
|
||||
# 构建实到索引: waybill_no → [piece_no, ...]（保留所有行，不去重）
|
||||
act_by_wb: dict[str, list[str]] = {}
|
||||
for wb, piece in act_rows:
|
||||
act_by_wb.setdefault(wb, []).append(piece)
|
||||
|
||||
stats = CompareStats()
|
||||
rows: list[UndeliveredRow] = []
|
||||
max_arrived = 0
|
||||
|
||||
for wb, handover_no, handover_pcs in exp_rows:
|
||||
handover_pcs = handover_pcs or 0
|
||||
if handover_pcs <= 0:
|
||||
continue
|
||||
|
||||
stats.waybill_count += 1
|
||||
stats.expected_pieces += handover_pcs
|
||||
|
||||
is_sf = cfg.has_sf and wb.startswith("SF")
|
||||
if is_sf:
|
||||
stats.sf_wb_count += 1
|
||||
|
||||
all_pieces = act_by_wb.get(wb, [])
|
||||
|
||||
if is_sf:
|
||||
# SF: 行计数，不去重（piece_no 是随机号码）
|
||||
arrived_cnt = len(all_pieces)
|
||||
arrived_list = list(all_pieces)
|
||||
else:
|
||||
# non-SF: 子单号去重
|
||||
unique_pieces = list(dict.fromkeys(all_pieces)) # 保序去重
|
||||
arrived_cnt = len(unique_pieces)
|
||||
arrived_list = unique_pieces
|
||||
|
||||
stats.arrived_pieces += arrived_cnt
|
||||
|
||||
if arrived_cnt >= handover_pcs:
|
||||
continue # 足额或溢到，不进差缺表
|
||||
|
||||
if arrived_cnt == 0:
|
||||
stats.full_miss += 1
|
||||
else:
|
||||
stats.part_miss += 1
|
||||
|
||||
if is_sf:
|
||||
stats.sf_undelivered += 1
|
||||
|
||||
max_arrived = max(max_arrived, arrived_cnt)
|
||||
rows.append(
|
||||
UndeliveredRow(
|
||||
handover_no=handover_no,
|
||||
waybill_no=wb,
|
||||
total_pieces=handover_pcs,
|
||||
arrived_pieces=arrived_cnt,
|
||||
arrived_list=arrived_list,
|
||||
is_sf=is_sf,
|
||||
)
|
||||
)
|
||||
|
||||
stats.undelivered_pieces = max(0, stats.expected_pieces - stats.arrived_pieces)
|
||||
stats.undelivered_wb = stats.full_miss + stats.part_miss
|
||||
|
||||
result = CompareResult(
|
||||
site=site,
|
||||
date=label,
|
||||
batches=batches,
|
||||
stats=stats,
|
||||
rows=rows,
|
||||
)
|
||||
|
||||
# 打印摘要
|
||||
print(
|
||||
f"[db_compare] {site} {label}: "
|
||||
f"batches={len(batches)}, "
|
||||
f"wb={stats.waybill_count}(SF:{stats.sf_wb_count}), "
|
||||
f"exp={stats.expected_pieces}, arr={stats.arrived_pieces}, "
|
||||
f"miss={stats.undelivered_pieces}, "
|
||||
f"miss_wb={stats.undelivered_wb}(full={stats.full_miss}, part={stats.part_miss})"
|
||||
)
|
||||
if stats.sf_undelivered:
|
||||
print(f" SF 差缺: {stats.sf_undelivered} 个运单")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ============================== Excel 输出 ==============================
|
||||
|
||||
|
||||
# 样式常量（与 compare.py 对齐）
|
||||
_FONT = "微软雅黑"
|
||||
_BLUE = "305496"
|
||||
|
||||
_HEADER_FILL = PatternFill("solid", fgColor=_BLUE)
|
||||
_HEADER_FONT = Font(name=_FONT, bold=True, color="FFFFFF", size=11)
|
||||
_BODY_FONT = Font(name=_FONT, size=10)
|
||||
_THIN = Side(style="thin", color="D9D9D9")
|
||||
_BORDER = Border(left=_THIN, right=_THIN, top=_THIN, bottom=_THIN)
|
||||
|
||||
|
||||
def write_result_excel(result: CompareResult, output_path: str | None = None) -> str:
|
||||
"""将比对结果写入 Excel 文件。
|
||||
|
||||
Args:
|
||||
result: compare_site_date 或 compare_site_batch 的返回值
|
||||
output_path: 输出路径，为 None 时自动生成：
|
||||
output/{站}-{日期}-未到数据.xlsx
|
||||
|
||||
Returns:
|
||||
实际写入的文件路径
|
||||
"""
|
||||
if output_path is None:
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
date_tag = result.date.replace(":", "-").replace("batch:", "batch-")
|
||||
output_path = os.path.join(
|
||||
OUTPUT_DIR, f"{result.site}-{date_tag}-未到数据.xlsx"
|
||||
)
|
||||
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = result.site
|
||||
|
||||
_write_sheet(ws, result)
|
||||
wb.save(output_path)
|
||||
print(f"[db_compare] Excel 已输出: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
def _write_sheet(ws, result: CompareResult):
|
||||
"""写单个站点的差缺明细 sheet。"""
|
||||
s = result.stats
|
||||
rows = result.rows
|
||||
|
||||
# 动态列: 交接单号 | 运单号 | 总件数 | 已到单号1 | 已到单号2 | ...
|
||||
max_arrived = max((len(r.arrived_list) for r in rows), default=0)
|
||||
columns = ["交接单号", "运单号", "总件数"] + [
|
||||
f"已到单号{i + 1}" for i in range(max_arrived)
|
||||
]
|
||||
|
||||
ws.sheet_view.showGridLines = False
|
||||
|
||||
# 表头
|
||||
ws.append(columns)
|
||||
for c in range(1, len(columns) + 1):
|
||||
cell = ws.cell(row=1, column=c)
|
||||
cell.fill = _HEADER_FILL
|
||||
cell.font = _HEADER_FONT
|
||||
cell.alignment = Alignment(horizontal="center", vertical="center")
|
||||
cell.border = _BORDER
|
||||
|
||||
# 数据行
|
||||
for row in rows:
|
||||
values = {
|
||||
"交接单号": row.handover_no,
|
||||
"运单号": row.waybill_no,
|
||||
"总件数": row.total_pieces,
|
||||
}
|
||||
for i, piece in enumerate(row.arrived_list):
|
||||
values[f"已到单号{i + 1}"] = piece
|
||||
ws.append([values.get(c, "") for c in columns])
|
||||
|
||||
# 格式
|
||||
for r in range(2, ws.max_row + 1):
|
||||
for c, col in enumerate(columns, start=1):
|
||||
cell = ws.cell(row=r, column=c)
|
||||
cell.font = _BODY_FONT
|
||||
cell.border = _BORDER
|
||||
if col == "总件数":
|
||||
cell.number_format = "#,##0"
|
||||
cell.alignment = Alignment(horizontal="right", vertical="center")
|
||||
else:
|
||||
cell.number_format = "@"
|
||||
|
||||
# 列宽
|
||||
for c, col in enumerate(columns, start=1):
|
||||
body_lens = [
|
||||
len(str(ws.cell(row=r, column=c).value or ""))
|
||||
for r in range(2, ws.max_row + 1)
|
||||
]
|
||||
width = min(max([len(str(col))] + body_lens) + 4, 36)
|
||||
ws.column_dimensions[ws.cell(row=1, column=c).column_letter].width = max(
|
||||
width, 12
|
||||
)
|
||||
|
||||
ws.freeze_panes = "A2"
|
||||
|
||||
|
||||
# ============================== 终端验证入口 ==============================
|
||||
|
||||
|
||||
def main():
|
||||
"""命令行验证入口：
|
||||
python -m inbound_verify.db_compare 顺心 2026-07-25
|
||||
"""
|
||||
import sys
|
||||
|
||||
site = sys.argv[1] if len(sys.argv) > 1 else "顺心"
|
||||
target_date = sys.argv[2] if len(sys.argv) > 2 else "2026-07-25"
|
||||
|
||||
result = compare_site_date(site, target_date)
|
||||
if result is None:
|
||||
print(f"{site} {target_date}: 无结果")
|
||||
return
|
||||
|
||||
print(f"\n=== {result.site} {result.date} 差缺明细 ===")
|
||||
print(f"涉及批次: {result.batches}")
|
||||
print(f"应到运单: {result.stats.waybill_count} (SF: {result.stats.sf_wb_count})")
|
||||
print(f"应到件数: {result.stats.expected_pieces}")
|
||||
print(f"实到件数: {result.stats.arrived_pieces}")
|
||||
print(f"未到件数: {result.stats.undelivered_pieces}")
|
||||
print(
|
||||
f"差缺运单: {result.stats.undelivered_wb} (完全未到: {result.stats.full_miss}, 部分未到: {result.stats.part_miss})"
|
||||
)
|
||||
if result.stats.sf_undelivered:
|
||||
print(f"SF 差缺: {result.stats.sf_undelivered}")
|
||||
|
||||
if result.rows:
|
||||
print(f"\n--- 差缺明细 (共 {len(result.rows)} 条) ---")
|
||||
for row in result.rows[:20]:
|
||||
sf = "[SF]" if row.is_sf else ""
|
||||
arrived_preview = row.arrived_list[:5]
|
||||
print(
|
||||
f" {sf} {row.waybill_no}: "
|
||||
f"应到{row.total_pieces}件, 实到{row.arrived_pieces}件"
|
||||
f" {f'已到: {arrived_preview}' if arrived_preview else ''}"
|
||||
)
|
||||
if len(result.rows) > 20:
|
||||
print(f" ... 还有 {len(result.rows) - 20} 条")
|
||||
|
||||
# 输出 Excel
|
||||
path = write_result_excel(result)
|
||||
print(f"\n结果文件: {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user