引入 PostgreSQL 持久化:到货核销数据入库管道
- 新增 db_store.py:解析 downloads/ 下各站应到/实到/未到 Excel,幂等 UPSERT 入库(统一核心列 + raw JSONB 兜底,单号按文本存) - 新增 schema.sql:三表(expected/actual/undelivered_record),隔离到专用 schema inbound_verify(CREATE SCHEMA + search_path) - 韵达实到业务清洗:抛弃「交接单号」为空行 + 按子单号去重 - 同步 config.example.yaml(postgres 配置)与 requirements.txt(psycopg[binary]) 本次仅手动入库管道;自动挂载到下载流程留待后续。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
54
schema.sql
Normal file
54
schema.sql
Normal file
@@ -0,0 +1,54 @@
|
||||
-- =====================================================================
|
||||
-- CQHXDB / 到货核销数据持久化 schema
|
||||
-- 设计:统一核心表 + JSONB raw 兜底(保留站点原始全列,一字段不丢)
|
||||
-- 幂等:业务唯一键 UPSERT,重复下载天然合并、零冗余
|
||||
-- 表:expected_record(运单级)/ actual_record(扫描件级)
|
||||
-- / undelivered_record(百世站点直供未到明细,子单级)
|
||||
-- =====================================================================
|
||||
|
||||
-- 隔离到专用 schema(不污染 public);表名无需再加前缀。
|
||||
CREATE SCHEMA IF NOT EXISTS inbound_verify;
|
||||
SET search_path TO inbound_verify;
|
||||
|
||||
-- 应到货物(运单级:一运单一行;按运单号去重 keep-first 后入库)
|
||||
CREATE TABLE IF NOT EXISTS expected_record (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
site TEXT NOT NULL, -- 站点:顺心 / 中通 / 韵达 / 安能
|
||||
waybill_no TEXT NOT NULL, -- 运单基号(业务唯一键,去重键)
|
||||
handover_no TEXT, -- 交接单号
|
||||
handover_pieces INTEGER, -- 交接件数(应到件数口径)
|
||||
order_pieces INTEGER, -- 录单件数
|
||||
business_date DATE, -- 业务日期(属性,非唯一键;读不到则 NULL)
|
||||
raw JSONB NOT NULL, -- 站点原始全列(key=原列名)
|
||||
ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (site, waybill_no)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_expected_site_date ON expected_record (site, business_date);
|
||||
|
||||
-- 实到货物(扫描件级:一扫描一行;每扫描一件系统生成一个单号)
|
||||
CREATE TABLE IF NOT EXISTS actual_record (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
site TEXT NOT NULL,
|
||||
waybill_no TEXT NOT NULL, -- 运单基号(由扫描单号推导)
|
||||
piece_no TEXT NOT NULL, -- 扫描单号 / 子单号(业务唯一键)
|
||||
scan_time TIMESTAMPTZ, -- 扫描时间(尽力解析,失败存 NULL,原始值在 raw)
|
||||
scan_site TEXT, -- 扫描网点
|
||||
raw JSONB NOT NULL,
|
||||
ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (site, piece_no)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_actual_waybill ON actual_record (site, waybill_no);
|
||||
CREATE INDEX IF NOT EXISTS idx_actual_scan_time ON actual_record (scan_time);
|
||||
|
||||
-- 应到未到(百世站点直供未到明细:子单级)
|
||||
CREATE TABLE IF NOT EXISTS undelivered_record (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
site TEXT NOT NULL, -- 百世
|
||||
waybill_no TEXT, -- 运单号
|
||||
piece_no TEXT, -- 子单号(业务唯一键)
|
||||
biz_type TEXT, -- 类型
|
||||
last_scan TEXT, -- 最新扫描记录
|
||||
raw JSONB NOT NULL,
|
||||
ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (site, piece_no)
|
||||
);
|
||||
Reference in New Issue
Block a user