Compare commits
36 Commits
7211846375
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bccf7cd396 | ||
|
|
c1bd53d832 | ||
|
|
3c32720985 | ||
|
|
53c71aeeac | ||
|
|
837264f7b0 | ||
|
|
eaf56c1c9e | ||
|
|
dc7653c256 | ||
|
|
3c7e9f2522 | ||
|
|
8521c200ab | ||
|
|
c6ad6a0ca2 | ||
|
|
541836fd1b | ||
|
|
95597fbb0c | ||
|
|
f66e6dd39e | ||
|
|
09e05f8dfc | ||
|
|
047036f46b | ||
|
|
21662d3944 | ||
|
|
4f0ef69739 | ||
|
|
5e2ef72afd | ||
|
|
fcb75643d0 | ||
|
|
cda764a305 | ||
|
|
e51aab2e0b | ||
|
|
bacccc43ab | ||
|
|
e240d92ad9 | ||
|
|
30e9203fca | ||
|
|
93e48118bf | ||
|
|
a68fd46c51 | ||
|
|
e2ae3acd68 | ||
|
|
f710622a3d | ||
|
|
d74a35844f | ||
|
|
82ab6d8be3 | ||
|
|
29edcbfdbb | ||
|
|
5e4889e845 | ||
|
|
0111734e9c | ||
|
|
615661c275 | ||
|
|
327f77a727 | ||
|
|
62a66467a9 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -42,6 +42,7 @@ desktop.ini
|
||||
downloads/
|
||||
output/
|
||||
state/
|
||||
logs/
|
||||
*.xlsx
|
||||
*.xls
|
||||
*.log
|
||||
|
||||
18
CLAUDE.md
18
CLAUDE.md
@@ -33,9 +33,9 @@ playwright install chromium
|
||||
.venv/Scripts/python.exe -m inbound_verify.cli.server
|
||||
# 或:inbound-verify-server
|
||||
|
||||
# DB CLI:建库 / 初始化 / 灌数据 / 全流程
|
||||
.venv/Scripts/python.exe -m inbound_verify.store createdb # 或 init | ingest | all
|
||||
# 或:inbound-verify-db createdb|init|ingest|all
|
||||
# DB CLI:建库 / 初始化 / 灌数据 / 全流程 / 单站单类
|
||||
.venv/Scripts/python.exe -m inbound_verify.store createdb # 或 init | ingest | ingest-one <site> <kind> | all
|
||||
# 或:inbound-verify-db createdb|init|ingest|ingest-one|all
|
||||
|
||||
# 单站点联调:在 config.yaml 设 debug.enabled=true + debug.target_site=顺心|百世|中通|韵达|安能
|
||||
# 网页站:只挂载该站;安能:只启动 Electron 应用。
|
||||
@@ -112,6 +112,18 @@ playwright install chromium
|
||||
`inbound_verify.compare`(菜单 [9])纯离线:读 `downloads/` 下各站应到/实到 xlsx,
|
||||
比对生成 `output/应到未到数据.xlsx`(汇总 + 各站明细)。
|
||||
|
||||
### 自动入库(下载成功后 → PostgreSQL)
|
||||
`dispatch_task` 下载成功后,在 `_record_business_date` 旁挂一个**尽力而为**钩子
|
||||
`_persist_to_db(site, kind)`(`runtime`):懒导入 `store`,调 `store.ingest_task(site, kind)`
|
||||
按 kind 幂等 UPSERT 进 PostgreSQL(expected/actual 各入其列;undelivered——百世入未到、
|
||||
4 站连入 expected+actual)。**绝不影响下载任务的成功判定**:所有写库/写状态都包 try/except,
|
||||
失败只告警。
|
||||
- 开关 `postgres.auto_ingest`(默认开)+ `connect_timeout_seconds`(兜 cpolar 抖动)。
|
||||
- 可见性:结果写 `state_store.ingest_state`(每站每类 ok/count/ingested_at/error),经
|
||||
`GET /status` 的 `ingest` 字段暴露。
|
||||
- 手动:`store` CLI `ingest-one <site> <kind>` 用同套路由单测(脱离下载)。
|
||||
- 注意:跑在 Playwright 线程、同步阻塞;cpolar 慢/断靠超时 + try/except 降级,不重试不补入。
|
||||
|
||||
### 路径
|
||||
`inbound_verify.paths` 把 `DOWNLOAD_DIR` / `CONFIG_PATH` / `BASE_DIR` 全部锚定到项目目录,
|
||||
**不依赖运行时 cwd**——别用相对路径或 `os.getcwd()`。
|
||||
|
||||
10
README.md
10
README.md
@@ -39,7 +39,7 @@ InboundVerify/
|
||||
│ ├── state_store.py # SQLite 状态持久化(state/state.db)
|
||||
│ ├── domain.py # 站点 / 文件名 / 列映射共享配置(单一来源,leaf)
|
||||
│ ├── compare.py # 全站点应到未到离线比对,输出 output/应到未到数据.xlsx
|
||||
│ ├── store.py # DB CLI 入口(createdb|init|ingest|all)
|
||||
│ ├── store.py # DB CLI 入口(createdb|init|ingest|ingest-one|all)
|
||||
│ ├── sites/ # 各站点模块(流程 + 重置 + 重试,自洽)
|
||||
│ │ ├── shunxin.py # 顺心(含双账号)
|
||||
│ │ ├── baishi.py # 百世
|
||||
@@ -137,6 +137,14 @@ inbound-verify
|
||||
[0] 退出
|
||||
```
|
||||
|
||||
### DB CLI(入库)
|
||||
|
||||
```bash
|
||||
# DB CLI:建库 / 初始化 / 灌数据 / 全流程 / 单站单类
|
||||
.venv/Scripts/python.exe -m inbound_verify.store createdb # 或 init | ingest | ingest-one <site> <kind> | all
|
||||
# 注:下载成功后会自动入库(postgres.auto_ingest,默认开);ingest-one 用于手动重灌指定站/类。
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 六、架构
|
||||
|
||||
@@ -46,7 +46,7 @@ zto:
|
||||
# 韵达快运 (https://ky-sso.yunda56.com)
|
||||
# ----------------------------------------------------------------------------
|
||||
yunda:
|
||||
# 自动登录的账号与密码(由 site_yunda.yunda_login 读取并填入登录表单)。
|
||||
# 自动登录的账号与密码(由 inbound_verify.sites.yunda.yunda_login 读取并填入登录表单)。
|
||||
# 留空时自动登录会填入空串导致登录失败,届时可在浏览器中改为手动登录。
|
||||
# 真实凭据仅写进被 .gitignore 忽略的 config.yaml,切勿提交示例值。
|
||||
username: "YOUR_USERNAME_HERE"
|
||||
@@ -58,9 +58,9 @@ yunda:
|
||||
# ----------------------------------------------------------------------------
|
||||
# 安能全网门户(Electron 桌面应用,非网页)
|
||||
# ----------------------------------------------------------------------------
|
||||
# 与其他站点不同:安能不由 main_router 用浏览器打开,而是以调试模式启动其
|
||||
# 与其他站点不同:安能不由浏览器打开,而是以调试模式启动其
|
||||
# Electron 可执行文件(自动选取一个空闲端口作为 --remote-debugging-port,避免端口冲突),
|
||||
# 启动后请在应用内手动登录,main_router 会自动轮询判断是否进入主页。
|
||||
# 启动后请在应用内手动登录,runtime 会自动轮询判断是否进入主页。
|
||||
anneng:
|
||||
# 【已废弃】下载日期改由 Web 前端/API 按站点设偏移(0=今天,1=昨天…,存 state.db),此项不再生效。
|
||||
query_days: 1
|
||||
|
||||
102
docs/2026-07-29-应到提交导出去重-实现总结.md
Normal file
102
docs/2026-07-29-应到提交导出去重-实现总结.md
Normal file
@@ -0,0 +1,102 @@
|
||||
# 应到数据「提交导出任务前」去重 — 实现总结
|
||||
|
||||
> 日期:2026-07-29
|
||||
> 范围:顺心 / 中通 / 韵达 / 安能 4 站**应到(expected)**数据;百世与实到不在本次范围。
|
||||
|
||||
## 一、运行机制
|
||||
|
||||
在周期 / 手动触发下载时,于**提交导出任务之前**按交接单号判断该批应到数据是否已落库,已落库则跳过,从源头消除重复下载与重复落库。
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
TRIG[周期调度 / 手动触发<br/>task_spec: site, kind, force] --> DISP[dispatch_task → 站点 download_impl]
|
||||
DISP --> LOAD{force 强制重下?}
|
||||
LOAD -- 是 --> EMPTY[existing = 空集]
|
||||
LOAD -- 否 --> QRY[查 PG expected_record.handover_no]
|
||||
QRY -- cpolar 失败 --> EMPTY
|
||||
QRY -- 成功 --> SET[existing = 已落库交接单号集合]
|
||||
EMPTY --> LOOP[遍历本次查询到的班次/交接单号]
|
||||
SET --> LOOP
|
||||
LOOP --> JUDGE{交接单号 ∈ existing?}
|
||||
JUDGE -- 是 → 已落库 --> SKIP[⏭️ 跳过:不提交导出<br/>不 append export_times]
|
||||
JUDGE -- 否 → 新单 --> EXP[提交导出任务 → 轮询下载 → 入库 UPSERT]
|
||||
SKIP --> DONE{全部处理完}
|
||||
EXP --> DONE
|
||||
DONE --> FINAL{本次提交了新任务?}
|
||||
FINAL -- 无 → 全跳过 --> BAIL[空兜底 return:不进下载轮询]
|
||||
FINAL -- 有 --> POLL[轮询导出任务管理页 → 下载 → 入库]
|
||||
```
|
||||
|
||||
**核心要点:**
|
||||
|
||||
- **去重数据源**:PostgreSQL `expected_record.handover_no`(已落库的权威记录),新增 `store.get_existing_handover_nos(site)` 查询。
|
||||
- **判断时机**:提交导出任务**之前**(循环内逐单判断),而非下载之后。
|
||||
- **安全降级**:PG 不可用 / `force=true` → `existing=空集` → 当作未落库 → 继续提交(**宁可重复、绝不漏**,UPSERT 兜底)。
|
||||
- **空兜底**:全部跳过时 `export_times` 为空 → 直接 `return`,不进下载轮询(避免下载数校验失败 / 空转超时)。
|
||||
- **force 开关**:前端 checkbox(默认关)→ `POST /tasks.force` → 一路透传到 impl;周期调度恒不 force。
|
||||
|
||||
## 二、force 强制重下透传链路
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
UI[前端 checkbox<br/>forceRedownload] --> POST["POST /api/tasks<br/>{site,kind,force}"]
|
||||
POST --> BFF[Next BFF 透传]
|
||||
BFF --> TS["task_spec<br/>{site,kind,force}"]
|
||||
TS --> DISP[dispatch_task]
|
||||
DISP --> HDR["handler(ctx, force)"]
|
||||
HDR --> DL["download(pg, force)"]
|
||||
DL --> IMPL["impl(pg, force)"]
|
||||
IMPL --> DEC{force?}
|
||||
DEC -- 是 --> EMPTY2["existing = 空集<br/>强制重下,跳过去重"]
|
||||
DEC -- 否 --> LOAD2[查 PG 加载 existing]
|
||||
```
|
||||
|
||||
> 周期调度(`_enqueue_fetch`)投递任务时不带 `force` → 默认不强制。
|
||||
|
||||
## 三、4 站点标识获取
|
||||
|
||||
| 站点 | 提交前标识 | 来源 |
|
||||
| --- | --- | --- |
|
||||
| 中通 / 韵达 / 安能 | 交接单号(原有代码已读取) | DOM 列 / CDP 复选框 |
|
||||
| 顺心 | 交接单号 `RTS\d{3}WJ\d+` | 点"运单列表"后从界面读取(方式1) |
|
||||
|
||||
> 顺心"班次号"业务上等同交接单号;4 站统一用交接单号(= DB `handover_no`)作去重键。
|
||||
|
||||
## 四、改动概览
|
||||
|
||||
**后端 InboundVerify:**
|
||||
|
||||
| 文件 | 改动 |
|
||||
| --- | --- |
|
||||
| `schema.sql` | +`idx_expected_handover` 索引 |
|
||||
| `inbound_verify/store.py` | +`get_existing_handover_nos(site)`(含 cpolar 降级) |
|
||||
| `inbound_verify/cli/server.py` | `TaskRequest.force` + 透传到 task_spec |
|
||||
| `inbound_verify/runtime.py` | `dispatch_task` + 所有 handler 透传 `force` 到 `download(impl)` |
|
||||
| `inbound_verify/sites/{zto,yunda,anneng,shunxin}.py` | `impl` 加 `force` + 提交导出前注入去重 + 空兜底 |
|
||||
| `inbound_verify/sites/baishi.py` | `force` 形参兼容 |
|
||||
|
||||
**前端 dashboard:**
|
||||
|
||||
| 文件 | 改动 |
|
||||
| --- | --- |
|
||||
| `app/page.tsx` | `forceRedownload` state + checkbox + `trigger`/`triggerPrimary`/`onTrigger` 透传 force |
|
||||
|
||||
> BFF `app/api/tasks/route.ts` 是 generic 透传,无需改动。
|
||||
|
||||
## 五、验证结论
|
||||
|
||||
4 站去重 + force 开关均实测通过:
|
||||
|
||||
| 站点 | 二次触发行为 | 结果 |
|
||||
| --- | --- | --- |
|
||||
| 中通 | `...801` 已入库 → 跳过 → 空兜底 return | 20s(vs 首次 59s) ✅ |
|
||||
| 顺心 | 双账号 4 班次全跳过(`RTS023WJ375478` 等) | ✅ |
|
||||
| 韵达 | `...82001` 已入库 → 跳过 → 原有空兜底 | ✅ |
|
||||
| 安能 | `4008242619171180544` 已入库 → 跳过 | **5s(vs 4 分钟)** ✅ |
|
||||
| force | `[去重] 强制重下,跳过去重` + 已入库的重新导出 | ✅ |
|
||||
|
||||
实施过程中的两个问题均已解决:
|
||||
1. **顺心 RTS 正则**:`RTS\d+` 遇字母 W 停(只抓 `RTS023`)→ 改 `RTS[A-Z0-9]+` 抓完整 `RTS023WJ375320`。
|
||||
2. **韵达 force 偶发失败**:韵达站点自身 UI 不稳定(`section iframe` 匹配到 2 个 + 弹窗遮挡),与去重/force 无关;换安能验证 force 成功。
|
||||
|
||||
静态检查:Black(py310)+ compileall + tsc 全绿。
|
||||
399
docs/2026-07-29-应到提交导出去重实施计划.md
Normal file
399
docs/2026-07-29-应到提交导出去重实施计划.md
Normal file
@@ -0,0 +1,399 @@
|
||||
# 应到数据「提交导出任务前」去重 实施计划 v2
|
||||
|
||||
> **执行约定:** 本仓库无 pytest,验证靠实跑站点流程 + DB 核对(见第 7 节)。
|
||||
> 本仓库约定 **不自动提交**;所有改动落地后等用户明确说"提交"再 commit/push。
|
||||
> 本次会话额外约定:未获用户明确指示前不动代码、不提交。
|
||||
|
||||
**Goal:** 在周期性自动落库场景下,于「提交导出任务」之前,按**交接单号**(顺心=运单列表界面里的交接单号)判断该批应到数据是否已落库,已落库则跳过提交导出任务,从源头消除重复下载与重复落库;并提供一个"强制重下"开关(默认关)兜底。
|
||||
|
||||
**Architecture:**
|
||||
- 去重数据源 = PostgreSQL `expected_record.handover_no`(已存在字段,权威);`store.py` 新增 `get_existing_handover_nos(site)`(含 cpolar 降级)。
|
||||
- 在 4 站**应到**下载循环内、提交导出动作之前注入"命中已落库则 `continue`"。
|
||||
- `force` 开关经 `task_spec` → `dispatch_task` → handler → 各站 `download(page, force)` → `impl(page, force)` 透传;`force=True` 时跳过去重。周期 job 默认不 force。
|
||||
|
||||
**Tech Stack:** Python 3.10+ / Playwright(网页 3 站)/ 裸 CDP(安能)/ psycopg / SQLite;前端 Next.js 16 + React 19。
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Python 一律 `.venv`;改完任何 `.py` 必须跑 `.venv/Scripts/python.exe -m black inbound_verify`。
|
||||
- 改完跑 `.venv/Scripts/python.exe -m py_compile inbound_verify` 自检。
|
||||
- **不自动提交/推送**(覆盖全局 auto-push 默认)。
|
||||
- **不动 `export_times` 时间容差(≤40s)匹配机制**(CLAUDE.md 约定)。
|
||||
- 安能 CDP 驱动,**绝不用 `Page.reload`**。
|
||||
- `config.yaml` 已 gitignore,不提交真实凭据。
|
||||
- 前端是 **Next.js 16(有 breaking changes)**,写前端代码前先查 `node_modules/next/dist/docs/`。
|
||||
- 行号基于 2026-07-29 快照,实现时以当前代码为准、就近定位。
|
||||
|
||||
## 1. 已定决策(v1 审核反馈)
|
||||
|
||||
| 决策 | 结论 |
|
||||
|---|---|
|
||||
| A. 去重数据源 | **查 PostgreSQL `expected_record.handover_no`** |
|
||||
| B. 范围 | **本次只做应到(expected)**;实到/百世不动 |
|
||||
| C. 强制开关 | **加 force 开关,默认不强制重下** |
|
||||
| 顺心标识 | **方式1:点"运单列表"后、点导出前,从运单列表界面读交接单号**(与其他 3 站统一用交接单号去重) |
|
||||
|
||||
## 2. 背景与问题根源
|
||||
|
||||
周期链路:`fetch_schedule(IntervalTrigger) → task_queue → worker → dispatch_task → handler → 提交导出+下载 → _persist_to_db(UPSERT)`。
|
||||
- DB 已幂等(`expected_record` 按 `(site, waybill_no)` UPSERT)。
|
||||
- 但 `dispatch_task` 调 handler 前**无"是否需要下载"判断**,周期触发重复"提交导出→下载→解析"。
|
||||
- 本方案在「提交导出任务」前按交接单号去重,从源头省掉重复下载。
|
||||
|
||||
## 3. 各站探索结论(注入点)
|
||||
|
||||
| 站点 | 文件 | 提交导出位置 | 提交前标识 | 来源 |
|
||||
|---|---|---|---|---|
|
||||
| 中通 | `sites/zto.py` | `zto_expected_download_impl` L264 | ✅ 已有 `handover_no`(L251) | 主表行 `td.nth(3)` 正则 18 位 |
|
||||
| 韵达 | `sites/yunda.py` | `yunda_expected_download_impl` L342 | ✅ 已有 `raw_no`(L291) | 列表行 `td.nth(1)` |
|
||||
| 安能 | `sites/anneng.py` | 主循环 L927(逐条) | ✅ 已有 `ewbs_no`(L907-913) | CDP 复选框 `ewbsListNo=` 正则 19 位 |
|
||||
| 顺心 | `sites/shunxin.py` | L319 点导出 | 🆕 方式1:L316 后读运单列表界面交接单号 | DOM 待实勘 |
|
||||
|
||||
**顺心方式1关键事实**(已探明):raw 里「班次号」「交接单号」都有,且一个交接单 = 一个班次 = 多条运单;交接单号即入库 `handover_no`,与另 3 站同键。
|
||||
|
||||
## 4. 文件结构
|
||||
|
||||
| 文件 | 改动 |
|
||||
|---|---|
|
||||
| `schema.sql` | 加 `expected_record(site, handover_no)` 索引 |
|
||||
| `inbound_verify/store.py` | 新增 `get_existing_handover_nos(site)` |
|
||||
| `inbound_verify/cli/server.py` | `TaskRequest` 加 `force`;`create_task` 透传 force(周期不 force) |
|
||||
| `inbound_verify/runtime.py` | `dispatch_task` 读 force 传 handler;所有 handler 加 `force` 形参并透传到 `download_func` |
|
||||
| `inbound_verify/sites/zto.py` | `download/impl` 加 `force`;应到循环注入去重 + 空兜底 |
|
||||
| `inbound_verify/sites/yunda.py` | 同上(空兜底已存在) |
|
||||
| `inbound_verify/sites/anneng.py` | 同上 |
|
||||
| `inbound_verify/sites/shunxin.py` | `download/impl` 加 `force`;方式1:点运单列表后读交接单号去重 + 退回 + 空兜底 |
|
||||
| `dashboard/app/page.tsx` | 加 `forceRedownload` state + checkbox;`trigger/triggerPrimary` 透传 force |
|
||||
|
||||
> 注:4 站的 actual(实到)`download` 入口也统一加 `force=False` 形参(接收但不用,仅让 `_web_handler` 的统一调用成立),actual impl 不改。
|
||||
|
||||
## 5. 任务分解
|
||||
|
||||
### Task 1:schema.sql 加索引
|
||||
|
||||
**Files:** Modify `schema.sql`(`idx_expected_site_date` 之后)
|
||||
|
||||
```sql
|
||||
CREATE INDEX IF NOT EXISTS idx_expected_handover ON expected_record (site, handover_no);
|
||||
```
|
||||
|
||||
**验证:** `.venv/Scripts/python.exe -m inbound_verify.store init`(幂等)。
|
||||
|
||||
---
|
||||
|
||||
### Task 2:store.py 新增查已落库交接单号集合
|
||||
|
||||
**Files:** Modify `inbound_verify/store.py`(`ingest_task` 之后)
|
||||
|
||||
**Produces:** `get_existing_handover_nos(site: str) -> set[str]`
|
||||
|
||||
```python
|
||||
def get_existing_handover_nos(site):
|
||||
"""查该站点已落库的交接单号集合(expected_record.handover_no)。
|
||||
供"提交导出任务前"去重:已落库的不再重复提交导出。
|
||||
PG 不可用(cpolar 抖动等)时返回空集 + 告警,调用方按"未确认存在"处理
|
||||
(继续提交导出,UPSERT 兜底,绝不因去重查询失败而漏数据)。"""
|
||||
try:
|
||||
with _connect(_load_pg_config()["dbname"]) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT handover_no FROM expected_record "
|
||||
"WHERE site=%s AND handover_no IS NOT NULL AND handover_no <> ''",
|
||||
(site,),
|
||||
)
|
||||
return {str(r[0]).strip() for r in cur.fetchall()}
|
||||
except Exception as e:
|
||||
print(f">> [去重] 查询已落库交接单号失败({site}),本次不去重: {e}")
|
||||
return set()
|
||||
```
|
||||
|
||||
**验证:** `.venv/Scripts/python.exe -c "from inbound_verify import store; print(len(store.get_existing_handover_nos('中通')))"` 不抛异常即过。
|
||||
|
||||
---
|
||||
|
||||
### Task 3:force 开关后端骨架(server + runtime)
|
||||
|
||||
让 `force` 从 `task_spec` 一路透传到各站 `download(page, force)`。`with_retry` 的 `flow` 是零参 lambda,force 经闭包捕获,**with_retry 不动**。
|
||||
|
||||
**(a) `server.py` TaskRequest + create_task:**
|
||||
|
||||
```python
|
||||
class TaskRequest(BaseModel):
|
||||
site: str
|
||||
kind: str
|
||||
force: bool = False # 新增:强制重下(忽略已落库去重),默认关
|
||||
```
|
||||
|
||||
```python
|
||||
# create_task 内
|
||||
task_queue.put((task_id, {"site": req.site, "kind": req.kind, "force": req.force}))
|
||||
```
|
||||
|
||||
> `_enqueue_fetch`(L133 周期投递)**保持不变**(不带 force → 默认 False)✅。
|
||||
|
||||
**(b) `runtime.py` dispatch_task 透传 force:**
|
||||
|
||||
```python
|
||||
def dispatch_task(ctx, task_spec):
|
||||
site = task_spec.get("site")
|
||||
kind = task_spec.get("kind")
|
||||
force = bool(task_spec.get("force", False)) # 新增
|
||||
...
|
||||
try:
|
||||
ret = handler(ctx, force) # 改:原 handler(ctx)
|
||||
```
|
||||
|
||||
**(c) `runtime.py` 所有 handler 加 force 形参:**
|
||||
|
||||
`_web_handler`:
|
||||
```python
|
||||
def handler(ctx, force=False):
|
||||
pg = ctx.pages_map[site]
|
||||
if isinstance(pg, list): # 顺心双账号
|
||||
return download_func(pg, foreground=ctx.foreground, force=force)
|
||||
if ctx.foreground:
|
||||
pg.bring_to_front()
|
||||
return download_func(pg, force=force)
|
||||
```
|
||||
|
||||
`_site_undelivered_handler`:
|
||||
```python
|
||||
def handler(ctx, force=False):
|
||||
exp_ok = TASK_HANDLERS[(site, "expected")](ctx, force) is not False
|
||||
act_ok = (TASK_HANDLERS[(site, "actual")](ctx, force) is not False) if exp_ok else False
|
||||
...
|
||||
```
|
||||
|
||||
安能 expected/actual 与 compare(签名兼容即可):
|
||||
```python
|
||||
("安能", "expected"): lambda ctx, force=False: anneng.anneng_expected_download(force=force),
|
||||
("安能", "actual"): lambda ctx, force=False: anneng.anneng_actual_download(force=force),
|
||||
("__compare__", "compare"): lambda ctx, force=False: (compare.main() or True),
|
||||
```
|
||||
|
||||
**验证:** `py_compile` 通过;服务重启后 `POST /tasks {site,kind,force:true}` 不报 TypeError(此时各站 `download` 的 force 形参由 Task 4-7 补齐,连续实施)。
|
||||
|
||||
---
|
||||
|
||||
### Task 4:中通 zto.py(download/impl 加 force + 去重)
|
||||
|
||||
**Files:** Modify `inbound_verify/sites/zto.py`
|
||||
|
||||
**(a) 入口与 impl 加 force(闭包透传,with_retry 不动):**
|
||||
```python
|
||||
def zto_expected_download(page, force=False):
|
||||
return with_retry(
|
||||
"中通", "应到",
|
||||
lambda: zto_expected_download_impl(page, force=force),
|
||||
lambda: zto_reset(page),
|
||||
)
|
||||
|
||||
def zto_expected_download_impl(page, force=False):
|
||||
...
|
||||
# zto_actual_download / zto_actual_download_impl 同样加 force=False 形参(actual 不用 force,仅兼容)
|
||||
```
|
||||
|
||||
**(b) 循环前加载已落库集合**(L241 print 之后、L243 `for` 之前):
|
||||
```python
|
||||
# 【去重】加载本站已落库交接单号;force=True 或查询失败时 existing=空集(不去重)
|
||||
if force:
|
||||
existing = set()
|
||||
print(">> [去重] 强制重下,跳过去重。")
|
||||
else:
|
||||
try:
|
||||
from inbound_verify import store
|
||||
existing = store.get_existing_handover_nos("中通")
|
||||
except Exception as _e:
|
||||
existing = set()
|
||||
print(f">> [去重] 加载失败,本次不去重: {_e}")
|
||||
```
|
||||
|
||||
**(c) 循环内命中跳过**(L252 print 之后、L254 `row.dblclick()` 之前):
|
||||
```python
|
||||
print(f" -> 当前交接单号:{handover_no}")
|
||||
if handover_no in existing:
|
||||
print(f" ⏭️ 交接单号 {handover_no} 已落库,跳过提交导出。")
|
||||
continue
|
||||
row.dblclick()
|
||||
```
|
||||
|
||||
**(d) 空列表兜底**(L305 循环后、进入 `_zto_poll_and_download_tasks` 之前):
|
||||
```python
|
||||
if not export_times:
|
||||
print(">> 本次无新交接单需导出(全部已落库或无数据),结束。")
|
||||
return
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5:韵达 yunda.py(同构)
|
||||
|
||||
**Files:** Modify `inbound_verify/sites/yunda.py`
|
||||
|
||||
(a) `yunda_expected_download(page, force=False)` + impl 加 force(actual 同理加形参);(b) 循环前加载 existing(同 Task 4b,站点"韵达");
|
||||
|
||||
**(c) 循环内命中跳过**(L291 `raw_no = ...` 之后、L293 `# 跳过已绑定的交接单` 之前):
|
||||
```python
|
||||
raw_no = current_row.locator("td").nth(1).inner_text().strip()
|
||||
if raw_no in existing:
|
||||
print(f" ⏭️ 交接单号 {raw_no} 已落库,跳过提交导出。")
|
||||
continue
|
||||
# 跳过已绑定的交接单
|
||||
bind_status = current_row.locator("td").nth(2).inner_text().strip()
|
||||
```
|
||||
|
||||
(d) 空列表兜底 **已存在**(L389-392),无需新增。
|
||||
|
||||
---
|
||||
|
||||
### Task 6:安能 anneng.py(CDP)
|
||||
|
||||
**Files:** Modify `inbound_verify/sites/anneng.py`
|
||||
|
||||
(a) `anneng_expected_download(force=False)` + impl 加 force(actual 同理);
|
||||
|
||||
**(b) 主流程加载 existing**(`anneng_expected_download_impl` 内、L907 收集 `target_ids` 前,与 `export_times = []`(L882)并列,同 Task 4b,站点"安能");
|
||||
|
||||
**(c) 主循环命中跳过**(L919 `for` 内、L920 print 之后、L921 `activate_tab` 之前):
|
||||
```python
|
||||
for i, ewbs_no in enumerate(target_ids, start=1):
|
||||
print(f" ⏳ [{i}/{len(target_ids)}] 交接单号 {ewbs_no}")
|
||||
if ewbs_no in existing:
|
||||
print(f" ⏭️ 交接单号 {ewbs_no} 已落库,跳过。")
|
||||
continue
|
||||
activate_tab(tab_cdp, "交接单信息")
|
||||
...
|
||||
```
|
||||
|
||||
**(d) 空列表兜底**(主循环后、进入 `poll_and_download_tasks` 之前):
|
||||
```python
|
||||
if not export_times:
|
||||
print(">> 本次无新交接单需导出(全部已落库或无数据),结束。")
|
||||
return
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7:顺心 shunxin.py(方式1 + 实勘)
|
||||
|
||||
顺心流程:L315 点"运单列表" → L316 等"运单查询"label(**运单列表界面**) → L319 点"导出"。方式1 在 L316 之后、L319 之前读交接单号。
|
||||
|
||||
**Step 1(实勘,不改业务逻辑):** 确认运单列表界面里**交接单号的 DOM 选择器**(哪个元素/列)。实勘方式二选一(待用户同意):
|
||||
- 方式 A:临时在 L316 后加调试打印(dump 运单列表界面关键 DOM 文本),跑一次顺心应到,从日志定位选择器,再删调试代码。
|
||||
- 方式 B:debug 模式(`config.yaml` debug.target_site=顺心,CDP 9223)单独挂载,用 Playwright CLI 观察。
|
||||
|
||||
**Step 2(注入,选择器 `<HANDOVER_SELECTOR>` 确认后替换):**
|
||||
|
||||
(a) 入口与 impl 加 force:
|
||||
```python
|
||||
def shunxin_expected_download(pages, foreground=True, force=False):
|
||||
return with_retry(
|
||||
"顺心", "应到",
|
||||
lambda: shunxin_expected_download_impl(pages, foreground=foreground, force=force),
|
||||
lambda: shunxin_reset(pages),
|
||||
)
|
||||
|
||||
def shunxin_expected_download_impl(pages, foreground=True, force=False):
|
||||
...
|
||||
# shunxin_actual_download / impl 同样加 force=False 形参(actual 不用)
|
||||
```
|
||||
|
||||
(b) 循环前加载 existing(同 Task 4b,站点"顺心";两账号共享同一 `existing`)。
|
||||
|
||||
**(c) 循环内:点运单列表 → 读交接单号 → 命中则退回跳过**(L315-316 之后、L319 点导出之前):
|
||||
```python
|
||||
for i in range(count):
|
||||
print(f" ⏳ 正在处理第 {i+1}/{count} 个班次...")
|
||||
|
||||
waybill_btns.nth(i).click()
|
||||
page.locator("label[title='运单查询']").wait_for(state="visible")
|
||||
|
||||
# 【方式1】运单列表界面已加载,读交接单号 → 已落库则退回列表跳过
|
||||
handover_no = page.locator("<HANDOVER_SELECTOR>").first.inner_text().strip()
|
||||
if handover_no in existing:
|
||||
print(f" ⏭️ 交接单号 {handover_no} 已落库,跳过提交导出。")
|
||||
page.get_by_role("tab", name="车辆点到").click() # 退回列表(复用 L336)
|
||||
page.wait_for_timeout(500)
|
||||
continue
|
||||
|
||||
# 4. 执行导出流程
|
||||
page.get_by_role("button", name="export 导出").click()
|
||||
...
|
||||
```
|
||||
|
||||
(d) 空列表兜底(L339 循环后、进入导出任务管理页轮询之前):
|
||||
```python
|
||||
if not export_times:
|
||||
print(">> 本次无新班次需导出(全部已落库或无数据),结束。")
|
||||
return
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8:前端 force 开关(dashboard)
|
||||
|
||||
**Files:** Modify `dashboard/app/page.tsx`(BFF `app/api/tasks/route.ts` 是 generic 透传,**不用改**)
|
||||
|
||||
**(a) 加 state(L21 附近):**
|
||||
```tsx
|
||||
const [forceRedownload, setForceRedownload] = useState(false);
|
||||
```
|
||||
|
||||
**(b) `trigger` 加 force 形参并写入 body(L27-50):**
|
||||
```tsx
|
||||
const trigger = useCallback(
|
||||
async (site: string, kind: string, label: string, force?: boolean) => {
|
||||
...
|
||||
body: JSON.stringify({ site, kind, force: !!force }),
|
||||
...
|
||||
},
|
||||
[refreshTasks],
|
||||
);
|
||||
```
|
||||
|
||||
**(c) `triggerPrimary` 透传(L58-63):**
|
||||
```tsx
|
||||
const triggerPrimary = useCallback(
|
||||
async (cfg: SiteConfig) => {
|
||||
await trigger(cfg.name, cfg.primaryKind, `${cfg.name}·获取未到`, forceRedownload);
|
||||
},
|
||||
[trigger, forceRedownload],
|
||||
);
|
||||
```
|
||||
|
||||
**(d) UI:在配置区/顶部加 checkbox(默认不勾):**
|
||||
```tsx
|
||||
<label className="inline-flex items-center gap-1 text-xs text-amber-700">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={forceRedownload}
|
||||
onChange={(e) => setForceRedownload(e.target.checked)}
|
||||
/>
|
||||
强制重新下载(忽略已落库去重)
|
||||
</label>
|
||||
```
|
||||
|
||||
> 仅"获取未到"主按钮透传 force;周期抓取不经前端、恒不 force。
|
||||
|
||||
**验证:** 前端勾选 → 触发 → 网络面板看到 POST `/api/tasks` body 含 `force:true`;后端日志 `强制重下,跳过去重`。
|
||||
|
||||
## 6. 关键注意事项(陷阱)
|
||||
|
||||
- **跳过的交接单号绝不 `export_times.append`**:否则 `len(export_times)` > 实际提交数 → 下载数校验失败。所有 `continue` 都在 append 之前。
|
||||
- **全部跳过时必须 `return`**:`export_times` 为空时不进导出任务管理页轮询。
|
||||
- **顺心方式1跳过要退回**:点进运单列表后命中已落库,需点"车辆点到"tab 退回再 `continue`(复用 L336)。
|
||||
- **PG 降级只防漏不防重**:查询失败 = 空集 = 当作未存在 = 继续提交;UPSERT 兜底。
|
||||
- **不动 `export_times` 时间容差匹配**。
|
||||
- actual/百世 `download` 只加 `force` 形参兼容,impl 不加去重。
|
||||
|
||||
## 7. 验证(无 pytest)
|
||||
|
||||
1. **单站联调**(`config.yaml` debug 单站):首次新单正常下载+入库;再触发同范围 → 已入库的全部 `⏭️ 跳过`,`export_times` 空,直接 return。
|
||||
2. **force 开关**:勾选"强制重下" → 已落库的也重新提交导出(日志 `强制重下,跳过去重`)。
|
||||
3. **DB 核对**:`SELECT site, handover_no, COUNT(*) FROM expected_record GROUP BY site, handover_no` 无翻倍。
|
||||
4. **cpolar 降级**:断 PG → `get_existing_handover_nos` 返回空集 + 告警,流程仍正常下载(不漏)。
|
||||
5. **Black + py_compile**;前端 `npm run build` 或 dev 热更无类型错。
|
||||
|
||||
## 8. 实施顺序与依赖
|
||||
|
||||
Task 1 → 2 → 3(骨架,此时各站 download 的 force 形参在 4-7 补)→ 4/5/6/7(各站,连续做完让链路自洽)→ 8(前端)。顺心 Task 7 的 Step1 实勘需在运行的服务上操作,实施时与用户协调时机。
|
||||
131
docs/2026-07-29-指定日期下载接口-design.md
Normal file
131
docs/2026-07-29-指定日期下载接口-design.md
Normal file
@@ -0,0 +1,131 @@
|
||||
# 指定日期下载接口(开发者)— 设计文档
|
||||
|
||||
> 日期:2026-07-29
|
||||
> 定位:面向开发者的 HTTP 接口,**不进前端**。提供"指定一个具体日期,下载该日应到 / 实到数据"的能力,用于补下历史数据。
|
||||
> 前置:中通跨月导航已实现并验证(见 `feat(zto): cross-month calendar navigation`)。
|
||||
|
||||
## 一、背景与目标
|
||||
|
||||
现状:下载日期由各站 **offset 偏移**(0=今天,1=昨天…,存 `state.db`,上限 `MAX_DATE_OFFSET=30`)决定,周期调度与手动触发都用 offset。无法指定一个具体日期。
|
||||
|
||||
目标:新增"指定日期"入口(开发者用),传一个 `YYYY-MM-DD` 日期,下载该日应到 / 实到数据。不替换 offset 机制,与之并存:传 date 用 date,不传走 offset。
|
||||
|
||||
## 二、范围
|
||||
|
||||
| 站点 | 支持指定日期 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| 顺心 / 中通 / 韵达 / 安能 | ✅ | 应到、实到均支持 |
|
||||
| 百世 | ❌ | 固定下载当天,传 date 返回 400 |
|
||||
|
||||
## 三、接口契约
|
||||
|
||||
复用 `POST /tasks`,body 新增可选字段 `date`(与 `force` 并列):
|
||||
|
||||
```json
|
||||
{ "site": "中通", "kind": "expected", "date": "2026-06-14" }
|
||||
```
|
||||
|
||||
- `date: Optional[str] = None`,格式 `YYYY-MM-DD`。
|
||||
- **优先级**:传 `date` 则本次用 date;不传则走站点 offset 配置(默认行为完全不变)。
|
||||
- `date` 与 `force` 可共存(指定日期 + 强制重下)。
|
||||
|
||||
### 合法性校验(仅在传了 date 时执行,失败返回 400)
|
||||
|
||||
1. **格式**:`datetime.strptime(date, "%Y-%m-%d")` 解析成功,否则 400。
|
||||
2. **范围**:`今天 - 90 天 ≤ date ≤ 今天`。
|
||||
- `date > 今天` → 400(未来日期;日历未来格子 `invalid` 物理上点不动,且不应下未来数据)。
|
||||
- `date < 今天 - 90 天` → 400(回溯上限 90 天)。
|
||||
3. **百世**:site=百世 且传 date → 400(固定当天)。
|
||||
|
||||
> 合法性校验落在 `POST /tasks`(`server.py` `create_task`),入队前拦截,非法请求不产生任务。
|
||||
|
||||
## 四、透传链路(与现有 `force` 完全对称)
|
||||
|
||||
```
|
||||
POST /tasks {site, kind, force, date}
|
||||
→ task_queue.put((tid, {site, kind, force, date}))
|
||||
→ dispatch_task(ctx, task_spec) # 读 task_spec["date"]
|
||||
→ handler(ctx, force, date) # _web_handler / 安能 lambda / _site_undelivered_handler
|
||||
→ impl(page, force, date) # 各站 download_impl
|
||||
```
|
||||
|
||||
- `_web_handler`:`handler(ctx, force=False, date=None)`,透传 `download_func(pg, force, date)`;顺心双账号透传 `(pages, foreground, force, date)`。
|
||||
- `_site_undelivered_handler`(未到):连下 expected + actual,**两个子任务共用同一个 date**。
|
||||
- 安能 lambda:`(ctx, force=False, date=None) → anneng_xxx_download(force=force, date=date)`。
|
||||
- **周期调度** `_enqueue_fetch` 投递的 task_spec 只有 `{site, kind}`(不带 date)→ 恒走 offset,**无需改动**。
|
||||
|
||||
## 五、各站 impl 改造(核心)
|
||||
|
||||
统一模式:**`target = parse(date) if date else (today − offset)`**。
|
||||
|
||||
### 中通(zto)—— 复用跨月算法
|
||||
|
||||
把 date 折算成 effective offset,复用现有 `target_time = today_time − offset*86400000` 与跨月翻页(`_zto_flip_to_target_month`),零额外 UI 逻辑:
|
||||
|
||||
```python
|
||||
def zto_expected_download_impl(page, force=False, date=None):
|
||||
...
|
||||
offset = state_store.get_offset("中通")
|
||||
if date:
|
||||
target_date = datetime.strptime(date, "%Y-%m-%d").date()
|
||||
offset = (datetime.now().date() - target_date).days
|
||||
# 后续 today_time / target_time / 跨月翻页 逻辑完全不变
|
||||
```
|
||||
|
||||
`zto_actual_download_impl` 同理(用 `("中通","actual")` offset)。expected / actual 两个 impl 都加 `date=None` 形参,`zto_expected_download` / `zto_actual_download` 公开入口同步加形参并透传。
|
||||
|
||||
### 韵达 / 顺心 / 安能 —— date 直接当 target
|
||||
|
||||
这三站 offset→日期是 `target = today − timedelta(days=offset)` 后填**字符串**到日期控件(非日历格子),指定日期只需替换 target 来源:
|
||||
|
||||
```python
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
```
|
||||
|
||||
后接的"填起始/截止日期字符串"逻辑完全不变。各站 expected / actual 入口与 impl 都加 `date=None` 形参。
|
||||
|
||||
- 韵达:`yunda_expected_download(_impl)` / `yunda_actual_download(_impl)`。
|
||||
- 顺心:`shunxin_expected_download(_impl)` / `shunxin_actual_download(_impl)`(双账号入口透传 date 到各账号 impl)。
|
||||
- 安能:`anneng_expected_download` / `anneng_actual_download`。
|
||||
|
||||
### 百世(baishi)—— 签名兼容
|
||||
|
||||
`baishi_download_undelivered_data(page, date=None)` 加 `date=None` 形参(**忽略**),仅为对齐 `_web_handler` 的统一透传签名;百世任务实际不会带 date(server 已拦截)。
|
||||
|
||||
## 六、业务日期快照
|
||||
|
||||
`_record_business_date(site, kind, date=None)`:有 date 则业务日期 = date,否则维持现状 `today − offset`。`dispatch_task` 把 `task_spec["date"]` 透传进去,保证状态盘 / 报告显示的"是哪天的数据"准确(不被 offset 算错)。
|
||||
|
||||
## 七、改动文件清单
|
||||
|
||||
| 文件 | 改动 |
|
||||
| --- | --- |
|
||||
| `inbound_verify/cli/server.py` | `TaskRequest.date` + `create_task` 合法性校验 + task_spec 透传 date |
|
||||
| `inbound_verify/runtime.py` | `_web_handler` / `_site_undelivered_handler` / 安能 lambda 透传 date;`dispatch_task` 读 date 透传给 handler 与 `_record_business_date`;`_record_business_date` 加 date |
|
||||
| `inbound_verify/sites/zto.py` | expected/actual 入口+impl 加 `date`;date→effective offset 复用跨月 |
|
||||
| `inbound_verify/sites/yunda.py` | expected/actual 入口+impl 加 `date`;date→target |
|
||||
| `inbound_verify/sites/shunxin.py` | 同上(双账号透传 date) |
|
||||
| `inbound_verify/sites/anneng.py` | expected/actual 加 `date`;date→target |
|
||||
| `inbound_verify/sites/baishi.py` | 加 `date=None` 形参兼容(忽略) |
|
||||
|
||||
## 八、验证计划
|
||||
|
||||
1. **接口校验**(curl/python urllib):
|
||||
- 合法 date(过去某日)→ 202,任务成功。
|
||||
- 未来日期 / 超 90 天 / 格式错 → 400。
|
||||
- 百世 + date → 400。
|
||||
- 不传 date → 走 offset(行为不变)。
|
||||
2. **各站实测**(指定一个过去日期触发任务):
|
||||
- 中通:跨月日期(已知 OK,复用已验证的跨月导航)。
|
||||
- 顺心 / 韵达 / 安能:实测其日期控件是否接受任意过去日期字符串;若控件是日历选择器需翻月,则按中通同法扩展(本轮发现则记录、必要时追加改动)。
|
||||
3. **业务日期快照**:下载后 `GET /status` 的 `*_business_date` == 指定 date。
|
||||
4. 改完跑 Black + `py_compile`。
|
||||
|
||||
## 九、非目标(YAGNI)
|
||||
|
||||
- 前端 UI(checkbox / 日期选择器)——开发者接口,不进前端。
|
||||
- 周期调度指定日期——周期恒走 offset。
|
||||
- 批量日期 / 日期范围下载——单次单日。
|
||||
682
docs/2026-07-29-指定日期下载接口-plan.md
Normal file
682
docs/2026-07-29-指定日期下载接口-plan.md
Normal file
@@ -0,0 +1,682 @@
|
||||
# 指定日期下载接口(开发者)Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** 给 `POST /tasks` 加可选 `date`(YYYY-MM-DD),支持顺心/中通/韵达/安能指定一个过去日期下载应到/实到数据;不传 date 时行为不变(走 offset)。
|
||||
|
||||
**Architecture:** 复用现有 `force` 透传链路加一个 `date` 字段:`TaskRequest.date` → `task_spec["date"]` → `dispatch_task` → `handler(ctx, force, date)` → 各站 `impl(page, force, date)`。各站把"算 target 日期"的来源从 `today - offset` 改为"有 date 用 date,否则 today - offset"。中通把 date 折算成 effective offset 以复用已验证的跨月翻页。
|
||||
|
||||
**Tech Stack:** Python 3.10、FastAPI、Playwright、SQLite(state_store)、PostgreSQL(store,可选入库)。
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **环境**:所有 Python 用 `D:/projects/LogisticsHubIPA/InboundVerify/.venv/Scripts/python.exe`(项目虚拟环境,无需激活)。PYTHONPATH 含 `InboundVerify` 根。
|
||||
- **格式化/自检(每改一个 .py 必做)**:`python -m black <file>`(target py310)+ `python -m py_compile <file>`。Black 若提示 "Python 3.10 cannot parse code formatted for 3.15",加 `--target-version py310`;"left unchanged" 即合格。
|
||||
- **无 pytest**:本项目无单元测试框架(见 `InboundVerify/CLAUDE.md`)。每个任务的"验证"= Black + py_compile;端到端(API 校验、各站实测)集中在 Task 8(需重启服务加载新代码)。
|
||||
- **submodule 工作流**:改动在 `InboundVerify` submodule(`dev` 分支)。每个 Task 末尾在 submodule 内 `git add <file> && git commit`。**push 到 origin/dev + 父仓库 bump** 统一在 Task 8(项目约定不自动 push,等用户确认;但 plan 内 commit 步骤照写)。
|
||||
- **顺序安全**:Task 1-5(站点)只给 impl 加 `date=None` 形参 + date 逻辑,**date 默认 None 时走原 offset 路径,向后兼容**;Task 6(runtime)才把 date 从 task_spec 透传进 impl;Task 7(server)才允许 date 入队。任一 Task 完成后系统均可正常运行。
|
||||
- **合法性边界**(来自 spec):`date` 传了才校验——格式 `YYYY-MM-DD`、`今天-90 ≤ date ≤ 今天`、百世不支持 date。非法返回 HTTP 400。
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| 文件 | 责任 | 本计划改动 |
|
||||
| --- | --- | --- |
|
||||
| `inbound_verify/sites/baishi.py` | 百世下载(固定当天) | 入口加 `date=None` 形参(忽略) |
|
||||
| `inbound_verify/sites/zto.py` | 中通下载(日历格子,跨月) | 入口+impl 加 `date`;date→effective offset 复用跨月 |
|
||||
| `inbound_verify/sites/yunda.py` | 韵达下载(日期字符串) | expected/actual 入口+impl 加 `date`;date→target |
|
||||
| `inbound_verify/sites/shunxin.py` | 顺心下载(双账号,日期字符串) | expected/actual 入口+impl 加 `date`;date→target |
|
||||
| `inbound_verify/sites/anneng.py` | 安能下载(CDP,日期字符串) | expected/actual 入口+impl 加 `date`;date→target |
|
||||
| `inbound_verify/runtime.py` | 任务派发/心跳共享核心 | handler/dispatch 透传 date;`_record_business_date` 用 date |
|
||||
| `inbound_verify/cli/server.py` | FastAPI 服务 | `TaskRequest.date` + 合法性校验 + task_spec 透传 |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: baishi.py — 入口加 date=None 形参(兼容)
|
||||
|
||||
**Files:**
|
||||
- Modify: `inbound_verify/sites/baishi.py:140`(`baishi_download_undelivered_data`)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `baishi_download_undelivered_data(page, force=False, date=None)` —— 后续 Task 6 的 `_web_handler` 会以 `download_func(pg, force=force, date=date)` 调用它,必须接受 `date` kwarg(百世忽略)。
|
||||
|
||||
- [ ] **Step 1: 改签名**
|
||||
|
||||
把 `def baishi_download_undelivered_data(page, force=False):` 改为:
|
||||
|
||||
```python
|
||||
def baishi_download_undelivered_data(page, force=False, date=None):
|
||||
"""百世:应到未到数据下载(固定当天;date 形参仅为对齐统一透传签名,忽略)。"""
|
||||
```
|
||||
|
||||
(函数体不动;`date` 不使用。)
|
||||
|
||||
- [ ] **Step 2: Black + py_compile**
|
||||
|
||||
```bash
|
||||
PY="D:/projects/LogisticsHubIPA/InboundVerify/.venv/Scripts/python.exe"
|
||||
F="D:/projects/LogisticsHubIPA/InboundVerify/inbound_verify/sites/baishi.py"
|
||||
"$PY" -m black "$F" && "$PY" -m py_compile "$F" && echo OK
|
||||
```
|
||||
Expected: black "left unchanged" 或 reformat 后通过;OK。
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" add inbound_verify/sites/baishi.py
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" commit -m "feat(baishi): accept date kwarg (ignored) for unified dispatch signature" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: zto.py — date 折算成 effective offset(复用跨月)
|
||||
|
||||
**Files:**
|
||||
- Modify: `inbound_verify/sites/zto.py` — `zto_expected_download`、`zto_expected_download_impl`、`zto_actual_download`、`zto_actual_download_impl`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `zto_expected_download(page, force=False, date=None)` / `zto_actual_download(page, force=False, date=None)`,impl 同签名。Task 6 的 `_web_handler` 以 `download_func(pg, force=force, date=date)` 调用。
|
||||
|
||||
- [ ] **Step 1: expected 入口加 date 并透传**
|
||||
|
||||
`def zto_expected_download(page, force=False):` 及其 `with_retry` 改为:
|
||||
|
||||
```python
|
||||
def zto_expected_download(page, force=False, date=None):
|
||||
"""中通:应到货物数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry(
|
||||
"中通",
|
||||
"应到",
|
||||
lambda: zto_expected_download_impl(page, force=force, date=date),
|
||||
lambda: zto_reset(page),
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: expected impl 加 date,date→effective offset**
|
||||
|
||||
`def zto_expected_download_impl(page, force=False):` 改签名加 `date=None`。其内"读取服务端日期偏移"段(`offset = state_store.get_offset("中通")` 与随后的 `print(...偏移...)`)改为:
|
||||
|
||||
```python
|
||||
# 读取服务端日期偏移(0=今天,1=昨天…),单日:起止同日
|
||||
offset = state_store.get_offset("中通")
|
||||
if date:
|
||||
# 指定日期:折算成相对今天的有效偏移,复用下方 target_time 计算与跨月翻月
|
||||
target_date = datetime.strptime(date, "%Y-%m-%d").date()
|
||||
offset = (datetime.now().date() - target_date).days
|
||||
print(f">> 正在设定查询日期: 指定日期 {date}(折算偏移 {offset})...")
|
||||
else:
|
||||
print(f">> 正在设定查询日期: 偏移 {offset}(0=今天)...")
|
||||
```
|
||||
|
||||
(其后的 `target_time = today_time - offset * 86400000` 与跨月翻月逻辑**不动**——date 经折算后走同一条路径。)
|
||||
|
||||
- [ ] **Step 3: actual 入口加 date 并透传**
|
||||
|
||||
`def zto_actual_download(page, force=False):` 及其 `with_retry` 改为:
|
||||
|
||||
```python
|
||||
def zto_actual_download(page, force=False, date=None):
|
||||
"""中通:实到货物数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry(
|
||||
"中通",
|
||||
"实到",
|
||||
lambda: zto_actual_download_impl(page, date=date),
|
||||
lambda: zto_reset(page),
|
||||
)
|
||||
```
|
||||
|
||||
> 注:`zto_actual_download_impl` 现签名 `(page)`(无 force,actual 不去重),Step 4 给它加 `date`。
|
||||
|
||||
- [ ] **Step 4: actual impl 加 date,date→effective offset**
|
||||
|
||||
`def zto_actual_download_impl(page):` 改为 `def zto_actual_download_impl(page, date=None):`。其内"# 2. 读取服务端日期偏移"段(`offset = state_store.get_offset("中通", "actual")` 与随后的 `print`)改为:
|
||||
|
||||
```python
|
||||
# 2. 读取服务端日期偏移(0=今天,1=昨天…),单日:起止同日
|
||||
offset = state_store.get_offset("中通", "actual")
|
||||
if date:
|
||||
target_date = datetime.strptime(date, "%Y-%m-%d").date()
|
||||
offset = (datetime.now().date() - target_date).days
|
||||
print(f">> 正在设定查询日期: 指定日期 {date}(折算偏移 {offset})...")
|
||||
else:
|
||||
print(f">> 正在设定查询日期: 偏移 {offset}(0=今天)...")
|
||||
```
|
||||
|
||||
(其后 `target_time = today_time - offset * 86400000` 与跨月翻月不动。)
|
||||
|
||||
- [ ] **Step 5: Black + py_compile**
|
||||
|
||||
```bash
|
||||
PY="D:/projects/LogisticsHubIPA/InboundVerify/.venv/Scripts/python.exe"
|
||||
F="D:/projects/LogisticsHubIPA/InboundVerify/inbound_verify/sites/zto.py"
|
||||
"$PY" -m black "$F" && "$PY" -m py_compile "$F" && echo OK
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" add inbound_verify/sites/zto.py
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" commit -m "feat(zto): support date arg via effective-offset (reuses cross-month nav)" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: yunda.py — date→target(expected + actual)
|
||||
|
||||
**Files:**
|
||||
- Modify: `inbound_verify/sites/yunda.py` — `yunda_expected_download(_impl)`、`yunda_actual_download(_impl)`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `yunda_expected_download(page, force=False, date=None)` / `yunda_actual_download(page, force=False, date=None)`,impl 同加 `date=None`。
|
||||
|
||||
- [ ] **Step 1: expected 入口透传 date**
|
||||
|
||||
```python
|
||||
def yunda_expected_download(page, force=False, date=None):
|
||||
...
|
||||
return with_retry(
|
||||
"韵达",
|
||||
"应到",
|
||||
lambda: yunda_expected_download_impl(page, force=force, date=date),
|
||||
lambda: yunda_reset(page),
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: expected impl 加 date + date→target**
|
||||
|
||||
`def yunda_expected_download_impl(page, force=False):` → `def yunda_expected_download_impl(page, force=False, date=None):`。其内日期段(`offset = state_store.get_offset("韵达")` 起几行)改为:
|
||||
|
||||
```python
|
||||
offset = state_store.get_offset("韵达")
|
||||
today = datetime.now()
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_ymd = f"{target.year}-{target.month}-{target.day}"
|
||||
start_date_ymd = target_ymd
|
||||
today_ymd = target_ymd
|
||||
print(f">> 设置查询日期: [{target_ymd}]({'指定 ' + date if date else f'偏移 {offset},0=今天'})")
|
||||
```
|
||||
|
||||
- [ ] **Step 3: actual 入口透传 date**
|
||||
|
||||
```python
|
||||
def yunda_actual_download(page, force=False, date=None):
|
||||
...
|
||||
return with_retry(
|
||||
"韵达",
|
||||
"实到",
|
||||
lambda: yunda_actual_download_impl(page, date=date),
|
||||
lambda: yunda_reset(page),
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: actual impl 加 date + date→target**
|
||||
|
||||
`def yunda_actual_download_impl(page):` → `def yunda_actual_download_impl(page, date=None):`。其内日期段改为:
|
||||
|
||||
```python
|
||||
offset = state_store.get_offset("韵达", "actual")
|
||||
today = datetime.now()
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
start_date = target # 单日范围:起止同日
|
||||
today = target # 让下方"截止时间"选择器也指向 target
|
||||
print(
|
||||
f">> 设置实到查询日期: [{target.year}-{target.month}-{target.day}]"
|
||||
f"({'指定 ' + date if date else f'偏移 {offset},0=今天'})"
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Black + py_compile**(同 Task 2 命令,文件换 yunda.py)
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" add inbound_verify/sites/yunda.py
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" commit -m "feat(yunda): support date arg (date takes precedence over offset)" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: shunxin.py — date→target(双账号透传)
|
||||
|
||||
**Files:**
|
||||
- Modify: `inbound_verify/sites/shunxin.py` — `shunxin_expected_download(_impl)`、`shunxin_actual_download(_impl)`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `shunxin_expected_download(pages, foreground=True, force=False, date=None)` / `shunxin_actual_download(pages, foreground=True, force=False, date=None)`。Task 6 的 `_web_handler` 以 `download_func(pg, foreground=ctx.foreground, force=force, date=date)` 调用(pg 是 page 列表)。
|
||||
|
||||
- [ ] **Step 1: expected 入口加 date 并向 impl 透传**
|
||||
|
||||
`def shunxin_expected_download(pages, foreground=True, force=False):` → 加 `date=None`。在函数内调用 `shunxin_expected_download_impl(page, out_tag=..., force=force)` 的位置,加上 `date=date`:
|
||||
|
||||
```python
|
||||
def shunxin_expected_download(pages, foreground=True, force=False, date=None):
|
||||
...
|
||||
# 对每个账号调用 impl 时透传 date:
|
||||
... shunxin_expected_download_impl(page, out_tag=归属地, force=force, date=date) ...
|
||||
```
|
||||
|
||||
> 执行者:用 Grep 定位 `shunxin_expected_download_impl(page,` 的调用处(在 `shunxin_expected_download` 函数体内,对每个账号调用一次),给每处加 `date=date`。签名行只加 `date=None`,其余函数体(归属地读取、去重校验、merge)不动。
|
||||
|
||||
- [ ] **Step 2: expected impl 加 date + date→target**
|
||||
|
||||
`def shunxin_expected_download_impl(page, out_tag="", force=False):` → 加 `date=None`。其内日期段(`offset = state_store.get_offset("顺心")` 起几行)改为:
|
||||
|
||||
```python
|
||||
offset = state_store.get_offset("顺心")
|
||||
today = datetime.now()
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_str = target.strftime("%Y-%m-%d")
|
||||
start_date_str = target_str
|
||||
today_str = target_str
|
||||
print(f">> 正在设置查询日期: [{target_str}]({'指定 ' + date if date else f'偏移 {offset},0=今天'})...")
|
||||
```
|
||||
|
||||
- [ ] **Step 3: actual 入口加 date 并透传**
|
||||
|
||||
`def shunxin_actual_download(pages, foreground=True, force=False):` → 加 `date=None`;调用 `shunxin_actual_download_impl(page, out_tag=...)` 处加 `date=date`。
|
||||
|
||||
- [ ] **Step 4: actual impl 加 date + date→target**
|
||||
|
||||
`def shunxin_actual_download_impl(page, out_tag=""):` → `def shunxin_actual_download_impl(page, out_tag="", date=None):`。其内日期段(`offset = state_store.get_offset("顺心", "actual")` 起几行)改为:
|
||||
|
||||
```python
|
||||
offset = state_store.get_offset("顺心", "actual")
|
||||
today = datetime.now()
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_str = target.strftime("%Y-%m-%d")
|
||||
start_date_str = target_str
|
||||
today_str = target_str
|
||||
print(f">> 正在设置查询日期: [{target_str}]({'指定 ' + date if date else f'偏移 {offset},0=今天'})...")
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Black + py_compile**(文件 shunxin.py)
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" add inbound_verify/sites/shunxin.py
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" commit -m "feat(shunxin): support date arg, propagate to both accounts" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: anneng.py — date→target(CDP,expected + actual)
|
||||
|
||||
**Files:**
|
||||
- Modify: `inbound_verify/sites/anneng.py` — `anneng_expected_download(_impl)`、`anneng_actual_download(_impl)`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `anneng_expected_download(force=False, date=None)` / `anneng_actual_download(force=False, date=None)`。Task 6 的 TASK_HANDLERS 安能项以 `lambda ctx, force=False, date=None: anneng.anneng_expected_download(force=force, date=date)` 调用。
|
||||
|
||||
- [ ] **Step 1: expected 入口 + impl 加 date,date→target**
|
||||
|
||||
```python
|
||||
def anneng_expected_download(force=False, date=None):
|
||||
return with_retry(
|
||||
"安能", "应到", lambda: anneng_expected_download_impl(force=force, date=date), anneng_reset
|
||||
)
|
||||
|
||||
|
||||
def anneng_expected_download_impl(force=False, date=None):
|
||||
```
|
||||
|
||||
expected impl 内日期段(`offset = state_store.get_offset("安能")` 起几行)改为:
|
||||
|
||||
```python
|
||||
offset = state_store.get_offset("安能")
|
||||
today = datetime.now()
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_str = f"{target.year}-{target.month:02d}-{target.day:02d}"
|
||||
start_str = target_str
|
||||
today_str = target_str
|
||||
print(f">> 查询日期: [{target_str}]({'指定 ' + date if date else f'偏移 {offset},0=今天'})")
|
||||
```
|
||||
|
||||
- [ ] **Step 2: actual 入口 + impl 加 date,date→target**
|
||||
|
||||
```python
|
||||
def anneng_actual_download(force=False, date=None):
|
||||
return with_retry("安能", "实到", lambda: anneng_actual_download_impl(date=date), anneng_reset)
|
||||
|
||||
|
||||
def anneng_actual_download_impl(date=None):
|
||||
```
|
||||
|
||||
actual impl 内日期段(`offset = state_store.get_offset("安能", "actual")` 起几行)改为:
|
||||
|
||||
```python
|
||||
offset = state_store.get_offset("安能", "actual")
|
||||
today = datetime.now()
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
start_str = f"{target.year}\{target.month:02d}/{target.day:02d} 00:00:00"
|
||||
end_str = f"{target.year}\{target.month:02d}/{target.day:02d} 23:59:59"
|
||||
print(f">> 扫描日期: [{start_str} 至 {end_str}]({'指定 ' + date if date else f'偏移 {offset},0=今天'})")
|
||||
```
|
||||
|
||||
> 注:actual 的 `start_str/end_str` 沿用现有 `{year}\{month}/{day}` 格式(含反斜杠,站点如此),只把 target 来源改成 date。
|
||||
|
||||
- [ ] **Step 3: Black + py_compile**(文件 anneng.py)
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" add inbound_verify/sites/anneng.py
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" commit -m "feat(anneng): support date arg (date takes precedence over offset)" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: runtime.py — 全链路透传 date + _record_business_date 用 date
|
||||
|
||||
**Files:**
|
||||
- Modify: `inbound_verify/runtime.py` — `_web_handler`、`_site_undelivered_handler`、`TASK_HANDLERS` 安能项、`dispatch_task`、`_record_business_date`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: Task 1-5 产出的各站 `*(..., date=None)` 签名。
|
||||
- Produces: `dispatch_task` 从 `task_spec["date"]` 读 date 透传给 handler 与 `_record_business_date`;handler 签名 `(ctx, force=False, date=None)`。
|
||||
|
||||
- [ ] **Step 1: _web_handler 透传 date**
|
||||
|
||||
```python
|
||||
def _web_handler(site, download_func):
|
||||
def handler(ctx, force=False, date=None):
|
||||
pg = ctx.pages_map[site]
|
||||
if isinstance(pg, list):
|
||||
# 顺心双账号:置顶与否交给 shunxin_download 在逐账号循环里按 foreground 决定
|
||||
return download_func(pg, foreground=ctx.foreground, force=force, date=date)
|
||||
if ctx.foreground:
|
||||
pg.bring_to_front()
|
||||
return download_func(pg, force=force, date=date)
|
||||
|
||||
return handler
|
||||
```
|
||||
|
||||
- [ ] **Step 2: _site_undelivered_handler 透传 date(连下 expected+actual 共用同一 date)**
|
||||
|
||||
```python
|
||||
def _site_undelivered_handler(site):
|
||||
def handler(ctx, force=False, date=None):
|
||||
exp_ok = TASK_HANDLERS[(site, "expected")](ctx, force, date) is not False
|
||||
act_ok = (
|
||||
(TASK_HANDLERS[(site, "actual")](ctx, force, date) is not False)
|
||||
if exp_ok
|
||||
else False
|
||||
)
|
||||
if exp_ok and act_ok:
|
||||
return compare.write_site_file(site)
|
||||
stale = os.path.join(DOWNLOAD_DIR, SITE_UNDELIVERED_FILE.format(name=site))
|
||||
if os.path.exists(stale):
|
||||
os.remove(stale)
|
||||
return False
|
||||
|
||||
return handler
|
||||
```
|
||||
|
||||
- [ ] **Step 3: TASK_HANDLERS 安能项透传 date**
|
||||
|
||||
```python
|
||||
("安能", "expected"): lambda ctx, force=False, date=None: anneng.anneng_expected_download(
|
||||
force=force, date=date
|
||||
),
|
||||
("安能", "actual"): lambda ctx, force=False, date=None: anneng.anneng_actual_download(
|
||||
force=force, date=date
|
||||
),
|
||||
```
|
||||
|
||||
(`__compare__` 项与百世/网页项不动——百世经 `_web_handler` 已透传 date,baishi 忽略。)
|
||||
|
||||
- [ ] **Step 4: dispatch_task 透传 date**
|
||||
|
||||
在 `dispatch_task` 内,把 `ret = handler(ctx, bool(task_spec.get("force", False)))` 改为:
|
||||
|
||||
```python
|
||||
ret = handler(
|
||||
ctx, bool(task_spec.get("force", False)), task_spec.get("date")
|
||||
)
|
||||
if ret is False:
|
||||
return (state_store.TASK_FAILED, "任务执行失败(重试耗尽)")
|
||||
_record_business_date(site, kind, task_spec.get("date"))
|
||||
_persist_to_db(site, kind)
|
||||
return (state_store.TASK_SUCCESS, None)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: _record_business_date 接受 date**
|
||||
|
||||
```python
|
||||
def _record_business_date(site, kind, date=None):
|
||||
"""下载成功后,把本次数据的业务日期快照写进状态库(供前端/报告显示「是哪天的数据」)。
|
||||
有 date 用 date;否则 = 下载当天 − 该数据对应的日期偏移。__compare__ 无数据概念,跳过。"""
|
||||
if site == "__compare__":
|
||||
return
|
||||
today = datetime.now().date()
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def _write(k, biz_or_off):
|
||||
# biz_or_off: int=offset(today−offset);str=已确定的业务日期(date)
|
||||
biz = (
|
||||
(today - timedelta(days=biz_or_off)).strftime("%Y-%m-%d")
|
||||
if isinstance(biz_or_off, int)
|
||||
else biz_or_off
|
||||
)
|
||||
try:
|
||||
state_store.set_data_state(
|
||||
site, k, ready=True, generated_at=now, business_date=biz
|
||||
)
|
||||
except Exception as e:
|
||||
print(f">> [状态] 写业务日期失败 {site}/{k}: {e}")
|
||||
|
||||
def off(kind_key):
|
||||
return state_store.get_offset(site, kind_key)
|
||||
|
||||
if kind == "expected":
|
||||
_write("expected", date if date else off("expected"))
|
||||
elif kind == "actual":
|
||||
_write("actual", date if date else off("actual"))
|
||||
elif site == "百世":
|
||||
_write("undelivered", 0)
|
||||
else: # 4 站 undelivered:连带补写 expected/actual/undelivered 三列
|
||||
_write("expected", date if date else off("expected"))
|
||||
_write("actual", date if date else off("actual"))
|
||||
_write("undelivered", date if date else off("expected"))
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Black + py_compile**
|
||||
|
||||
```bash
|
||||
PY="D:/projects/LogisticsHubIPA/InboundVerify/.venv/Scripts/python.exe"
|
||||
F="D:/projects/LogisticsHubIPA/InboundVerify/inbound_verify/runtime.py"
|
||||
"$PY" -m black "$F" && "$PY" -m py_compile "$F" && echo OK
|
||||
```
|
||||
|
||||
- [ ] **Step 7: 冒烟(date=None 行为不变)**
|
||||
|
||||
服务仍跑旧代码,但 runtime 模块可独立 import 校验:
|
||||
|
||||
```bash
|
||||
PYTHONPATH="D:/projects/LogisticsHubIPA/InboundVerify" "$PY" -c "import inbound_verify.runtime as r; import inspect; print('handler', inspect.signature(r._web_handler('中通', lambda *a, **k: None))); print('rec', inspect.signature(r._record_business_date))"
|
||||
```
|
||||
Expected: handler 含 `(ctx, force=False, date=None)`;rec 含 `(site, kind, date=None)`。
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" add inbound_verify/runtime.py
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" commit -m "feat(runtime): propagate date through dispatch chain and business-date snapshot" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: server.py — TaskRequest.date + 合法性校验 + task_spec
|
||||
|
||||
**Files:**
|
||||
- Modify: `inbound_verify/cli/server.py` — 顶部 import、`TaskRequest`、`create_task`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `POST /tasks` 接受 `{site, kind, force, date}`;合法 date 入队为 `task_spec["date"]`(YYYY-MM-DD),非法返回 400。
|
||||
|
||||
- [ ] **Step 1: 顶部 import 加 timedelta**
|
||||
|
||||
`from datetime import datetime` → `from datetime import datetime, timedelta`
|
||||
|
||||
- [ ] **Step 2: TaskRequest 加 date**
|
||||
|
||||
```python
|
||||
class TaskRequest(BaseModel):
|
||||
site: str
|
||||
kind: str
|
||||
force: bool = False
|
||||
date: Optional[str] = None # YYYY-MM-DD;指定则下载该日数据,否则走站点 offset
|
||||
```
|
||||
|
||||
- [ ] **Step 3: create_task 加合法性校验 + task_spec 透传 date**
|
||||
|
||||
```python
|
||||
@app.post("/tasks")
|
||||
def create_task(req: TaskRequest):
|
||||
"""提交任务 {site, kind, force, date?} → 入队,返回 task_id。"""
|
||||
if not worker_state["ready"]:
|
||||
raise HTTPException(
|
||||
status_code=409, detail="后端尚未就绪,请等待各站点登录完成后再操作"
|
||||
)
|
||||
if (req.site, req.kind) not in TASK_HANDLERS:
|
||||
raise HTTPException(status_code=400, detail=f"无效任务: {req.site}/{req.kind}")
|
||||
# 指定日期合法性校验(仅在传了 date 时)
|
||||
if req.date:
|
||||
try:
|
||||
target_date = datetime.strptime(req.date, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"date 格式非法,需 YYYY-MM-DD: {req.date}"
|
||||
)
|
||||
today = datetime.now().date()
|
||||
if target_date > today:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"date 不可为未来日期: {req.date}"
|
||||
)
|
||||
if target_date < today - timedelta(days=90):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"date 超出 90 天回溯上限: {req.date}"
|
||||
)
|
||||
if req.site == "百世":
|
||||
raise HTTPException(
|
||||
status_code=400, detail="百世固定下载当天,不支持指定日期"
|
||||
)
|
||||
task_id = state_store.create_task(req.site, req.kind)
|
||||
spec = {"site": req.site, "kind": req.kind, "force": req.force}
|
||||
if req.date:
|
||||
spec["date"] = req.date
|
||||
task_queue.put((task_id, spec))
|
||||
return {"task_id": task_id}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Black + py_compile**
|
||||
|
||||
```bash
|
||||
PY="D:/projects/LogisticsHubIPA/InboundVerify/.venv/Scripts/python.exe"
|
||||
F="D:/projects/LogisticsHubIPA/InboundVerify/inbound_verify/cli/server.py"
|
||||
"$PY" -m black "$F" && "$PY" -m py_compile "$F" && echo OK
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" add inbound_verify/cli/server.py
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" commit -m "feat(server): add date field to POST /tasks with legality validation" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 8: 端到端验证 + 收尾(push / 父 bump)
|
||||
|
||||
**Files:** 无代码改动;验证 + 提交推送。
|
||||
|
||||
- [ ] **Step 1: 重启服务加载全部新代码**
|
||||
|
||||
停掉旧服务进程,重启(中通调试模式,config.yaml 已是中通):
|
||||
|
||||
```bash
|
||||
PYTHONPATH="D:/projects/LogisticsHubIPA/InboundVerify" "D:/projects/LogisticsHubIPA/InboundVerify/.venv/Scripts/python.exe" -m inbound_verify.cli.server
|
||||
```
|
||||
(后台运行;等中通登录就绪。)
|
||||
|
||||
- [ ] **Step 2: 合法性校验端到端(API 400/202)**
|
||||
|
||||
用 Python urllib(避免 curl 中文编码问题)逐一验证,期望:
|
||||
|
||||
| 请求 | 期望 |
|
||||
| --- | --- |
|
||||
| `{site:"中通", kind:"expected", date:"2026-06-14"}` | 202 + task_id |
|
||||
| `date:"2099-01-01"`(未来) | 400 "不可为未来日期" |
|
||||
| `date:"2020-01-01"`(超 90 天) | 400 "超出 90 天回溯上限" |
|
||||
| `date:"2026/06/14"`(格式错) | 400 "格式非法" |
|
||||
| `{site:"百世", kind:"undelivered", date:"2026-06-14"}` | 400 "百世…不支持指定日期" |
|
||||
| `{site:"中通", kind:"expected"}`(不传 date) | 202(走 offset,行为不变) |
|
||||
|
||||
```bash
|
||||
"D:/projects/LogisticsHubIPA/InboundVerify/.venv/Scripts/python.exe" - <<'PYEOF'
|
||||
import json, urllib.request, urllib.error
|
||||
def post(body):
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request("http://127.0.0.1:8000/tasks", data=data,
|
||||
headers={"Content-Type":"application/json"}, method="POST")
|
||||
try:
|
||||
print(body, "->", urllib.request.urlopen(req, timeout=10).read().decode())
|
||||
except urllib.error.HTTPError as e:
|
||||
print(body, "->", e.code, e.read().decode())
|
||||
post({"site":"中通","kind":"expected","date":"2099-01-01"})
|
||||
post({"site":"中通","kind":"expected","date":"2020-01-01"})
|
||||
post({"site":"中通","kind":"expected","date":"2026/06/14"})
|
||||
post({"site":"百世","kind":"undelivered","date":"2026-06-14"})
|
||||
PYEOF
|
||||
```
|
||||
|
||||
- [ ] **Step 3: 中通跨月 date 实测(已知 OK)**
|
||||
|
||||
触发 `{site:"中通", kind:"expected", date:"2026-06-14"}`,观察 worker 日志:出现 `指定日期 2026-06-14(折算偏移 …)` + `偏移日期跨月,正在向前翻月导航` + 查询/下载成功。任务 `success`。
|
||||
|
||||
- [ ] **Step 4: 顺心/韵达/安能 date 实测**
|
||||
|
||||
对顺心/韵达/安能各触发一个 expected `date`(取一个近 1 周内的过去日期,确保站点有数据且控件能接受)。观察日志:`指定日期 …` + 正常查询下载。**若某站日期控件不接受字符串而需日历翻月**,记录现象,按中通同法(_zto_flip 模式)追加改动(可能产生新 Task)。
|
||||
|
||||
- [ ] **Step 5: 业务日期快照校验**
|
||||
|
||||
下载后 `curl -s http://127.0.0.1:8000/status`,确认对应站 `expected_business_date` == 指定 date。
|
||||
|
||||
- [ ] **Step 6: push submodule dev + 父仓库 bump**
|
||||
|
||||
```bash
|
||||
git -C "D:/projects/LogisticsHubIPA/InboundVerify" push origin dev
|
||||
git -C "D:/projects/LogisticsHubIPA" add InboundVerify
|
||||
git -C "D:/projects/LogisticsHubIPA" commit -m "chore: bump InboundVerify submodule (date-specific download API)" -m "Co-Authored-By: Claude <noreply@anthropic.com>"
|
||||
git -C "D:/projects/LogisticsHubIPA" push origin master
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review(plan 作者自检)
|
||||
|
||||
1. **Spec 覆盖**:接口契约→Task 7;透传链路→Task 6;中通 effective offset→Task 2;韵达/顺心/安能 date→target→Task 3/4/5;百世兼容→Task 1;业务日期快照→Task 6 Step 5;周期调度(不动)→无需 task(spec 明示);合法性校验→Task 7;验证→Task 8。✅ 全覆盖。
|
||||
2. **占位符**:无 TBD/TODO;每步含完整代码或精确命令。✅
|
||||
3. **类型/签名一致**:`date=None` 贯穿 server→runtime→sites;actual impl(yunda/shunxin/anneng)原本无 force,本计划只加 `date`,与 runtime 调用 `download_func(pg, force=, date=)` / 安能 `lambda(ctx,force,date)` 一致;shunxin 双账号 `(pages, foreground, force, date)` 与 `_web_handler` 的 list 分支一致。✅
|
||||
4. **顺序安全**:Task 1-5 向后兼容(date=None 走 offset);Task 6 启用透传但 server 未传 date(仍 None);Task 7 启用 date。✅
|
||||
332
docs/2026-07-31-四站点差缺对比逻辑审查报告.md
Normal file
332
docs/2026-07-31-四站点差缺对比逻辑审查报告.md
Normal file
@@ -0,0 +1,332 @@
|
||||
# 四站点差缺对比逻辑审查报告
|
||||
|
||||
> 审查日期:2026-07-31
|
||||
> 审查范围:顺心、中通、韵达、安能 四个站点的应到 vs 实到差缺对比逻辑
|
||||
> 排除:百世(站点直供未到明细,不参与四站比对)
|
||||
|
||||
---
|
||||
|
||||
## 一、比对算法总览(四站共用)
|
||||
|
||||
`compare.py:process()` 对四个站点执行**完全相同**的算法步骤。站点间的差异仅由 `domain.py:STATIONS` 配置注入——列名映射 + 实到单号解析器。
|
||||
|
||||
```
|
||||
步骤1: 读应到Excel → 按运单号去重keep-first → 构建 {运单号 → (交接单号, 交接件数=n)}
|
||||
步骤2: 读实到Excel → 站点专用解析器 → 构建 {运单基号 → {已到单号集合}}
|
||||
步骤3: 逐运单比对
|
||||
arrived_cnt >= n → 足额到货,跳过
|
||||
arrived_cnt == 0 → 完全未到
|
||||
0 < arrived < n → 部分未到
|
||||
步骤4: 产出未到明细(交接单号 | 运单号 | 总件数 | 已到单号1 | 已到单号2 | ...)
|
||||
```
|
||||
|
||||
### 核心口径
|
||||
|
||||
| 指标 | 口径 |
|
||||
|------|------|
|
||||
| 应到件数 | **交接件数**(非录单件数);按运单号去重 keep-first |
|
||||
| 实到件数 | 单号去重计数(每扫描一件=一个单号) |
|
||||
| 未到件数 | max(0, 应到件数 − 实到件数) |
|
||||
| 未到率 | 未到件数 ÷ 应到件数 |
|
||||
|
||||
### 未到明细输出约定
|
||||
|
||||
- 仅列出**短少运单**(实到 < 应到)
|
||||
- 列出该运单**实际已到的单号**(已到单号1, 已到单号2, ...)
|
||||
- **不编造缺件子单号**——实到扫描顺序号乱序,无法反推缺了哪个顺序号
|
||||
|
||||
### 统计指标
|
||||
|
||||
| 指标 | 含义 |
|
||||
|------|------|
|
||||
| 运单数 | 应到运单去重数 |
|
||||
| 应到件 | Σ 交接件数 |
|
||||
| 已到件 | Σ 实到单号去重数 |
|
||||
| 未到件 | max(0, 应到件 − 已到件) |
|
||||
| 涉及运单 | 存在短少的运单数 |
|
||||
| 完全未到 | 整单零到货运单数 |
|
||||
| 部分未到 | 部分缺件运单数 |
|
||||
|
||||
---
|
||||
|
||||
## 二、四站配置对照
|
||||
|
||||
`domain.py:STATIONS` — 所有差异集中于此配置表,比对核心代码不感知站点差异。
|
||||
|
||||
| 维度 | 中通 | 顺心 | 韵达 | 安能 |
|
||||
|------|------|------|------|------|
|
||||
| 应到文件 | `中通-应到货物数据.xlsx` | `顺心-应到货物数据.xlsx` | `韵达-应到货物数据.xlsx` | `安能-应到货物数据.xlsx` |
|
||||
| 实到文件 | `中通-实到货物数据.xlsx` | `顺心-实到货物数据.xlsx` | `韵达-实到货物数据.xlsx` | `安能-实到货物数据.xlsx` |
|
||||
| 应到-运单号列 | `运单号` | `运单号` | `运单号` | `运单号` |
|
||||
| 应到-件数列 | `交接件数` | `交接件数` | `交接件数` | `交接件数` |
|
||||
| 应到-交接单号列 | `交接单号` | `交接单号` | `交接单号` | `交接单号` |
|
||||
| 实到-基号列 | —(从复合串推导) | `运单号` | **`主单号`** | **`所属单号`** |
|
||||
| 实到-单号列 | `运单号`(复合串) | `子单号` | `子单号` | `扫描单号` |
|
||||
| 解析器 | `arrived_pieces_zhongtong` | `arrived_pieces_by_cols` | `arrived_pieces_by_cols` | `arrived_pieces_by_cols` |
|
||||
|
||||
---
|
||||
|
||||
## 三、逐站点详细分析
|
||||
|
||||
### 3.1 中通(ZTO)
|
||||
|
||||
#### 业务逻辑
|
||||
|
||||
实到货物数据中的「运单号」为复合串,由三部分构成:
|
||||
|
||||
```
|
||||
┌──────────┬────────────┬──────────┐
|
||||
│ 运单号 │ 录单件数 │ 顺序号 │
|
||||
│ (12位) │ (4位) │ (4位) │
|
||||
└──────────┴────────────┴──────────┘
|
||||
总长 20 位
|
||||
|
||||
示例: 330953527953 0001 0001
|
||||
├─ 运单号 ─┤├录单┤├顺序┤
|
||||
```
|
||||
|
||||
- **运单号(12位)**: 与应到货物数据中的运单号对齐
|
||||
- **录单件数(4位)**: 该运单在系统中的录单总件数,0占位
|
||||
- **顺序号(4位)**: 0占位,如 `0001`, `0002`, `0003`, `0004`
|
||||
|
||||
对比逻辑:
|
||||
1. 从应到数据取运单号 + 交接件数(**非录单件数**)
|
||||
2. 从实到数据取复合串,掐尾8位得运单基号,完整串为子运单号
|
||||
3. 按运单基号分组,子运单号去重得实到件数
|
||||
4. 实到件数 < 交接件数 → 差缺
|
||||
|
||||
> **重要**: 录单件数仅作参考。举例:某运单录单件数=4、交接件数=2,实到最多出现2条数据。如果只出现了1条,我们只知道差缺了,但**无法判断具体差缺了哪一件**(顺序号乱序)。
|
||||
|
||||
#### 代码实现
|
||||
|
||||
`domain.py:17-26` — 实到解析器:
|
||||
|
||||
```python
|
||||
def arrived_pieces_zhongtong(df):
|
||||
res = defaultdict(set)
|
||||
for v in df["运单号"]:
|
||||
v = str(v).strip()
|
||||
if len(v) > 8 and v[-4:].isdigit():
|
||||
res[v[:-8]].add(v) # 基号=前12位, 已到单号=完整20位复合串
|
||||
return res
|
||||
```
|
||||
|
||||
`domain.py:48-56` — 站点配置:
|
||||
|
||||
```python
|
||||
{
|
||||
"name": "中通",
|
||||
"exp_qty": "交接件数", # 应到件数口径:交接件数(非录单件数)
|
||||
"exp_wb": "运单号",
|
||||
"exp_jd": "交接单号",
|
||||
"arrived_pieces": arrived_pieces_zhongtong,
|
||||
"columns": ["交接单号", "运单号", "总件数"],
|
||||
}
|
||||
```
|
||||
|
||||
#### 对齐情况:✅ 对齐
|
||||
|
||||
代码实现与业务逻辑一致。`v[:-8]` 掐尾8位得12位运单基号,保留完整复合串作为已到单号——不解析、不推断录单件数和顺序号的具体含义。
|
||||
|
||||
---
|
||||
|
||||
### 3.2 安能(Anneng)
|
||||
|
||||
#### 业务逻辑
|
||||
|
||||
与中通相同的差缺对比逻辑。
|
||||
|
||||
安能实到数据同样为复合串,结构:`运单号(12位) + 录单件数(4位) + 顺序号(4位)`(20位)。
|
||||
|
||||
与中通的关键区别:安能实到表有**独立的「所属单号」列**(干净运单基号),无需像中通那样从复合串掐尾8位推导基号。
|
||||
|
||||
#### 代码实现
|
||||
|
||||
`domain.py:76-86`:
|
||||
|
||||
```python
|
||||
{
|
||||
"name": "安能",
|
||||
"arrived_pieces": arrived_pieces_by_cols("所属单号", "扫描单号"),
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
安能使用 `arrived_pieces_by_cols` 而非 `arrived_pieces_zhongtong`——直接从「所属单号」列读基号、从「扫描单号」列读完整单号,效果等价。
|
||||
|
||||
| 差异点 | 中通 | 安能 |
|
||||
|--------|------|------|
|
||||
| 实到基号来源 | 从复合串解析(`v[:-8]`) | 直接读「所属单号」列 |
|
||||
| 实到单号来源 | 复合串本身(「运单号」列) | 「扫描单号」列 |
|
||||
| 解析器 | `arrived_pieces_zhongtong` | `arrived_pieces_by_cols` |
|
||||
| 最终产出 | `{基号 → {完整单号集合}}` | 相同 |
|
||||
|
||||
#### 数据库验证
|
||||
|
||||
```
|
||||
piece_no=61003282264500140014 → waybill_no=610032822645 (12位), total=0014, seq=0014
|
||||
```
|
||||
|
||||
#### 对齐情况:✅ 对齐
|
||||
|
||||
---
|
||||
|
||||
### 3.3 顺心(Shunxin)⚠️
|
||||
|
||||
#### 业务逻辑
|
||||
|
||||
顺心站点需区分两类运单:
|
||||
|
||||
**A. 非SF开头运单(占 97%):**
|
||||
|
||||
实到「子单号」结构为两部分:
|
||||
|
||||
```
|
||||
┌──────────┬──────────┐
|
||||
│ 运单号 │ 顺序号 │
|
||||
│ (不定长) │ (3位) │
|
||||
└──────────┴──────────┘
|
||||
|
||||
示例: S71623721115 001
|
||||
├─ 运单号 ──┤├顺序┤
|
||||
|
||||
注意:顺心子单号无录单件数部分(仅两部分)
|
||||
```
|
||||
|
||||
对比时从实到取「子单号」列,按「运单号」分组,子单号去重得实到件数。
|
||||
|
||||
**B. SF开头运单(占 3%):**
|
||||
|
||||
SF订单的「子单号」为**随机号码**(非由运单号衍生),不能用于差缺推导。
|
||||
|
||||
对比逻辑:
|
||||
1. 在实到数据中按「运单号」字段查找,统计出现次数
|
||||
2. 出现次数 < 交接件数 → 差缺
|
||||
3. 将找到的子单号(虽随机但可以列出来)填入「已到单号」列
|
||||
|
||||
SF订单的差缺判定:**只基于交接件数与实到运单号出现次数的比较**,不依赖子单号的结构解析。
|
||||
|
||||
#### 代码实现
|
||||
|
||||
`domain.py:57-65`:
|
||||
|
||||
```python
|
||||
{
|
||||
"name": "顺心",
|
||||
"arrived_pieces": arrived_pieces_by_cols("运单号", "子单号"),
|
||||
}
|
||||
```
|
||||
|
||||
**SF 与非 SF 没有任何区分处理。** 所有运单走同一条路径。
|
||||
|
||||
#### 数据库验证
|
||||
|
||||
**非SF(正常):**
|
||||
```
|
||||
子单号=S71623721115001 → 运单号=S71623721115 + 后缀=001 ✅
|
||||
子单号=S71934073996002 → 运单号=S71934073996 + 后缀=002 ✅
|
||||
```
|
||||
|
||||
**SF(异常):**
|
||||
```
|
||||
运单号=SF1225002296515 的两条实到记录:
|
||||
子单号=SF2025318183224 (随机SF号码)
|
||||
子单号=SF1225002296515 (与运单号相同)
|
||||
```
|
||||
数据中有 10 个SF运单存在多条实到记录。
|
||||
|
||||
#### 对齐情况:⚠️ 部分对齐,SF特殊逻辑缺失
|
||||
|
||||
| 检查项 | 代码现状 | 业务要求 |
|
||||
|--------|----------|----------|
|
||||
| 非SF处理 | ✅ `arrived_pieces_by_cols("运单号", "子单号")` | 一致 |
|
||||
| 非SF子单号结构 | ✅ 运单号 + 顺序号(两部分) | 一致 |
|
||||
| SF处理 | ❌ 与非SF完全一致,使用子单号去重 | **不能**使用子单号,只按运单号行数计数 |
|
||||
| 功能影响 | 子单号虽随机但值唯一,按目前逻辑也能正确去重计数 | 但语义不正确——SF子单号不由运单号衍生 |
|
||||
|
||||
---
|
||||
|
||||
### 3.4 韵达(Yunda)❌
|
||||
|
||||
#### 业务逻辑
|
||||
|
||||
**去重规则:** 韵达实到数据存在重复行(同一子单号出现两次)。去重依据为「交接单号」字段:
|
||||
- **保留**交接单号为**空**的行
|
||||
- **丢弃**交接单号**非空**的行
|
||||
|
||||
**子单号结构:** 两部分——单号 + 顺序号(无录单件数部分)。
|
||||
|
||||
```
|
||||
┌──────────┬──────────┐
|
||||
│ 主单号 │ 顺序号 │
|
||||
│ (不定长) │ (4位) │
|
||||
└──────────┴──────────┘
|
||||
|
||||
示例: 713326603 0003
|
||||
├─主单号─┤├顺序┤
|
||||
```
|
||||
|
||||
**对比方式:** 与中通/安能同——按「主单号」分组,「子单号」去重得实到件数,与交接件数比对。
|
||||
|
||||
#### 代码实现
|
||||
|
||||
`store.py:316-320`(入库过滤):
|
||||
|
||||
```python
|
||||
if site == "韵达":
|
||||
# 韵达业务清洗:抛弃「交接单号」为空的行(派件/签收等其他扫描无交接单号),
|
||||
# 再按子单号去重(一件多扫只留一条;清洗后子单号已天然唯一,drop 为保险)。
|
||||
df = df[df["交接单号"].astype(str).str.strip() != ""] # ← 保留非空
|
||||
df = df.drop_duplicates(subset=[cm["piece"]], keep="last")
|
||||
```
|
||||
|
||||
`domain.py:67-75`(比对配置):
|
||||
|
||||
```python
|
||||
{
|
||||
"name": "韵达",
|
||||
"exp_wb": "运单号",
|
||||
"arrived_pieces": arrived_pieces_by_cols("主单号", "子单号"),
|
||||
}
|
||||
```
|
||||
|
||||
#### 对齐情况:❌ 交接单号过滤逻辑完全相反
|
||||
|
||||
| 检查项 | 代码现状 | 业务要求 |
|
||||
|--------|----------|----------|
|
||||
| 交接单号过滤 | 保留 `!= ""`(**非空**) | 保留 `== ""`(**空**) |
|
||||
| 子单号结构 | ✅ `7133266030003` = wb`713326603` + seq`0003` | 一致 |
|
||||
| 实到解析 | ✅ `arrived_pieces_by_cols("主单号", "子单号")` | 一致 |
|
||||
| compare.py 过滤 | ❌ **无过滤**,所有行参与比对 | 需要过滤 |
|
||||
|
||||
**影响分析:**
|
||||
|
||||
1. `store.py` 过滤反了——入库时留下了错误的数据集
|
||||
2. `compare.py` 完全没有交接单号过滤——如果原始 Excel 中同时存在空和非空行,比对阶段会全部读入导致重复计数
|
||||
3. 当前数据库中韵达 3483 条记录全部为非空交接单号——说明当前 Excel 数据中空交接单号行偏少或不存在,但这不改变逻辑错误
|
||||
|
||||
---
|
||||
|
||||
## 四、差异汇总
|
||||
|
||||
| # | 站点 | 问题 | 严重程度 | 影响范围 |
|
||||
|---|------|------|----------|----------|
|
||||
| 1 | **韵达** | 交接单号过滤反了:`!= ""` 应改为 `== ""` | ❌ 严重 | `store.py:319` + `compare.py` 需新增过滤 |
|
||||
| 2 | **顺心** | SF运单无特殊处理,与非SF混用子单号 | ⚠️ 中等 | `domain.py` 需新增SF判断分支 |
|
||||
| 3 | **中通** | 录单件数0占位描述与实际数据完全一致 | ✅ 无影响 | 代码不依赖此区分 |
|
||||
|
||||
---
|
||||
|
||||
## 五、代码位置索引
|
||||
|
||||
| 逻辑 | 文件 | 行号 |
|
||||
|------|------|------|
|
||||
| 单站比对 `process()` | `compare.py` | 61-143 |
|
||||
| 站点配置 `STATIONS` | `domain.py` | 46-87 |
|
||||
| 中通实到解析器 | `domain.py` | 17-26 |
|
||||
| 通用实到解析器 | `domain.py` | 29-42 |
|
||||
| 单站未到文件写入 | `compare.py` | 258-273 |
|
||||
| 全量汇总报告 | `compare.py` | 293-324 |
|
||||
| 未到触发编排 | `runtime.py` | 478-497 |
|
||||
| 韵达入库过滤(需修) | `store.py` | 316-320 |
|
||||
| 顺心实到配置(需修) | `domain.py` | 57-65 |
|
||||
264
docs/2026-07-31-顺心DB差缺对比实施计划.md
Normal file
264
docs/2026-07-31-顺心DB差缺对比实施计划.md
Normal file
@@ -0,0 +1,264 @@
|
||||
# 顺心 DB 差缺对比 — 实施计划
|
||||
|
||||
> 日期:2026-07-31
|
||||
> 目标:将顺心站点差缺对比从 Excel 读取改为 PostgreSQL 查询,并修正 SF 运单特殊处理逻辑
|
||||
|
||||
---
|
||||
|
||||
## 一、背景
|
||||
|
||||
### 当前状态(Excel 方式)
|
||||
|
||||
```
|
||||
compare.py:process("顺心")
|
||||
├── 读 downloads/顺心-应到货物数据.xlsx
|
||||
├── 读 downloads/顺心-实到货物数据.xlsx
|
||||
├── arrived_pieces_by_cols("运单号", "子单号") ← SF/non-SF 无区分
|
||||
└── 产出 {站}-未到数据.xlsx + 统计 dict
|
||||
```
|
||||
|
||||
### 需要解决的两个问题
|
||||
|
||||
1. **从 Excel 切换到 DB**:数据已持久化到 PostgreSQL,比对应直接从 DB 查询
|
||||
2. **顺心 SF 运单特殊处理**:SF 运单的子单号为随机号码,不能用于去重计数,应使用行计数
|
||||
|
||||
---
|
||||
|
||||
## 二、数据结构
|
||||
|
||||
### PostgreSQL 表
|
||||
|
||||
**expected_record**(关键列):
|
||||
|
||||
| 列 | 类型 | 说明 |
|
||||
|----|------|------|
|
||||
| site | TEXT | 站点 |
|
||||
| waybill_no | TEXT | 运单号(唯一键之一,SF 以 "SF" 开头) |
|
||||
| handover_no | TEXT | 交接单号(批次标识) |
|
||||
| handover_pieces | INTEGER | 交接件数(应到口径) |
|
||||
| order_pieces | INTEGER | 录单件数(参考) |
|
||||
| business_date | DATE | 下载目标日期 |
|
||||
|
||||
**actual_record**(关键列):
|
||||
|
||||
| 列 | 类型 | 说明 |
|
||||
|----|------|------|
|
||||
| site | TEXT | 站点 |
|
||||
| waybill_no | TEXT | 运单基号(关联 expected_record) |
|
||||
| piece_no | TEXT | 扫描单号(non-SF:运单号+顺序号;SF:随机号码) |
|
||||
| scan_time | TIMESTAMPTZ | 扫描时间(可靠,当天数据=当天扫描) |
|
||||
|
||||
### SF 数据特征(已验证)
|
||||
|
||||
- 顺心 actual_record 中 SF 运单:148 条
|
||||
- `piece_no == waybill_no`:86 条(58%)
|
||||
- `piece_no != waybill_no`:62 条(42%)← 随机 SF 号码
|
||||
- SF 运单 expected:99 条,分布在 31 个交接批次中
|
||||
|
||||
---
|
||||
|
||||
## 三、算法设计
|
||||
|
||||
### 核心思路:以实到为锚,通过交接单号反推批次
|
||||
|
||||
```
|
||||
输入: site="顺心", date="2026-07-25"
|
||||
|
||||
Step 1 — 取实到锚点
|
||||
SELECT DISTINCT waybill_no FROM actual_record
|
||||
WHERE site='顺心' AND scan_time::date = '2026-07-25'
|
||||
|
||||
Step 2 — 反推交接批次
|
||||
SELECT DISTINCT handover_no FROM expected_record
|
||||
WHERE site='顺心'
|
||||
AND waybill_no IN (Step 1 的运单集合)
|
||||
|
||||
Step 3 — 展开批次全量应到
|
||||
SELECT waybill_no, handover_no, handover_pieces
|
||||
FROM expected_record
|
||||
WHERE site='顺心'
|
||||
AND handover_no IN (Step 2 的交接单号集合)
|
||||
|
||||
Step 4 — 取批次全量实到
|
||||
SELECT waybill_no, piece_no FROM actual_record
|
||||
WHERE site='顺心'
|
||||
AND waybill_no IN (Step 3 的运单集合)
|
||||
|
||||
Step 5 — 逐运单比对
|
||||
for each waybill in Step 3:
|
||||
if waybill_no LIKE 'SF%':
|
||||
arrived_cnt = COUNT(*) ← 行计数,不去重
|
||||
else:
|
||||
arrived_cnt = COUNT(DISTINCT piece_no) ← 子单号去重
|
||||
if arrived_cnt < handover_pieces → 差缺
|
||||
```
|
||||
|
||||
### SF vs non-SF 处理差异
|
||||
|
||||
| | non-SF | SF |
|
||||
|------|--------|-----|
|
||||
| piece_no 含义 | 运单号 + 顺序号(可推导) | 随机 SF 号码(无推导意义) |
|
||||
| 实到计数方式 | `COUNT(DISTINCT piece_no)` | `COUNT(*)`(行计数) |
|
||||
| 已到单号列表 | 列出去重后的子单号 | 列出所有 piece_no(含重复) |
|
||||
|
||||
### 统计指标
|
||||
|
||||
| 指标 | 公式 |
|
||||
|------|------|
|
||||
| 运单数 | Step 3 去重运单数 |
|
||||
| 应到件 | Σ handover_pieces |
|
||||
| 已到件 | Σ arrived_cnt |
|
||||
| 未到件 | max(0, 应到件 − 已到件) |
|
||||
| 涉及运单 | arrived_cnt < handover_pieces 的运单数 |
|
||||
| 完全未到 | arrived_cnt = 0 的运单数 |
|
||||
| 部分未到 | 0 < arrived_cnt < handover_pieces 的运单数 |
|
||||
| 未到率 | 未到件 ÷ 应到件 |
|
||||
|
||||
### 边界情况覆盖
|
||||
|
||||
| 情况 | 覆盖方式 |
|
||||
|------|----------|
|
||||
| 同日多批次 | Step 2 查出全部涉及的 handover_no |
|
||||
| 跨天到达(延迟) | Step 4 不限 scan_time,历史扫描全计入 |
|
||||
| 溢到(实到 > 应到) | arrived_cnt >= n 跳过,不进差缺表 |
|
||||
| 完全沉默批次 | 一件未扫 = 实到无锚点,该批次不会被触发——在首次有扫描那天被纳入 |
|
||||
| SF 子单号重复 | 用 COUNT(*) 而非 COUNT(DISTINCT),不会漏计 |
|
||||
|
||||
---
|
||||
|
||||
## 四、模块设计
|
||||
|
||||
### 新增文件
|
||||
|
||||
**`inbound_verify/db_compare.py`** — DB 比对引擎(纯 PostgreSQL + Python)
|
||||
|
||||
```python
|
||||
# 核心函数签名
|
||||
|
||||
def compare_site_date(site: str, date: str) -> CompareResult | None:
|
||||
"""对指定站点和日期执行 DB 差缺比对。
|
||||
|
||||
返回 CompareResult(stats + undelivered_rows),
|
||||
当天无实到数据时返回 None。
|
||||
"""
|
||||
|
||||
def compare_site_batch(site: str, handover_no: str) -> CompareResult | None:
|
||||
"""按指定交接单号执行全批次比对(不依赖实到锚点)。"""
|
||||
```
|
||||
|
||||
**数据类型**:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CompareResult:
|
||||
stats: dict # 统计指标
|
||||
rows: list[dict] # 差缺明细行
|
||||
batches: list[str] # 涉及的交接批次
|
||||
|
||||
@dataclass
|
||||
class UndeliveredRow:
|
||||
handover_no: str # 交接单号
|
||||
waybill_no: str # 运单号
|
||||
total_pieces: int # 总件数(=交接件数)
|
||||
arrived_pieces: int # 已到件数
|
||||
arrived_list: list[str] # 已到单号列表
|
||||
is_sf: bool # 是否 SF 运单
|
||||
```
|
||||
|
||||
### 修改文件
|
||||
|
||||
**`inbound_verify/cli/server.py`** — 新增 API 端点
|
||||
|
||||
```python
|
||||
@app.post("/compare")
|
||||
def run_compare(req: CompareRequest):
|
||||
"""DB 比对:{site, date} → 返回差缺结果"""
|
||||
|
||||
@app.get("/compare/{site}/{date}")
|
||||
def get_compare(site: str, date: str):
|
||||
"""查询某站点某日的差缺结果(缓存)"""
|
||||
```
|
||||
|
||||
### 现有文件保持不动
|
||||
|
||||
- `compare.py` — 保留不动,Excel 比对继续可用
|
||||
- `domain.py` — 可能需要新增 DB 版站点配置(或复用现有)
|
||||
- `runtime.py` — 暂不改动,`_site_undelivered_handler` 仍走 Excel 路径
|
||||
|
||||
---
|
||||
|
||||
## 五、实施步骤
|
||||
|
||||
### Phase 1 — `db_compare.py` 核心引擎
|
||||
|
||||
- [ ] 新建 `inbound_verify/db_compare.py`
|
||||
- [ ] 实现 `compare_site_date("顺心", date)`
|
||||
- [ ] SF/non-SF 分支处理
|
||||
- [ ] 返回 `CompareResult`
|
||||
- [ ] 终端手动验证(直接调函数,打印结果)
|
||||
|
||||
### Phase 2 — API 端点
|
||||
|
||||
- [ ] 在 `server.py` 新增 `POST /compare`
|
||||
- [ ] `CompareRequest { site, date }`
|
||||
- [ ] 调用 `db_compare.compare_site_date()`
|
||||
- [ ] 返回 JSON:stats + undelivered rows
|
||||
- [ ] HTTP 验证:curl 调 `/compare` 对比不同日期结果
|
||||
|
||||
### Phase 3 — Excel 输出(可选)
|
||||
|
||||
- [ ] `db_compare` 生成 Excel 报告(复用现有 `compare.py` 的 openpyxl 样式)
|
||||
- [ ] 输出到 `output/顺心-{date}-未到数据.xlsx`
|
||||
- [ ] 或者只输出 JSON,前端自行渲染
|
||||
|
||||
### Phase 4 — 替换 undelivered 任务流
|
||||
|
||||
- [ ] `runtime.py` 新增 `_db_undelivered_handler`
|
||||
- [ ] 下载完成后不再调 Excel 比对,改调 DB 比对
|
||||
- [ ] 逐步替换 `TASK_HANDLERS` 中的顺心 undelivered handler
|
||||
|
||||
### Phase 5 — 扩展到中通/韵达/安能
|
||||
|
||||
- [ ] 各站适配(主要是 piece_no 去重方式差异)
|
||||
- [ ] 中通:`COUNT(DISTINCT piece_no)`,无 SF 问题
|
||||
- [ ] 韵达:同上
|
||||
- [ ] 安能:同上
|
||||
|
||||
---
|
||||
|
||||
## 六、测试策略
|
||||
|
||||
### 手工验证(Phase 1)
|
||||
|
||||
```python
|
||||
# 终端直接调
|
||||
from inbound_verify.db_compare import compare_site_date
|
||||
result = compare_site_date("顺心", "2026-07-25")
|
||||
print(result.stats)
|
||||
# 对比基于 Excel 版的 compare.process("顺心") 结果
|
||||
```
|
||||
|
||||
### API 验证(Phase 2)
|
||||
|
||||
```bash
|
||||
curl -X POST http://127.0.0.1:8000/compare \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"site":"顺心","date":"2026-07-25"}'
|
||||
```
|
||||
|
||||
### 回归验证
|
||||
|
||||
- 新 DB 比对结果 vs 旧 Excel 比对结果(同一份数据)
|
||||
- SF 运单的 arrived_cnt 对比:DB 版(COUNT(*))vs Excel 版(COUNT DISTINCT piece_no)
|
||||
- 确认 SF 运单不再被漏计
|
||||
|
||||
---
|
||||
|
||||
## 七、风险与注意事项
|
||||
|
||||
| 风险 | 缓解 |
|
||||
|------|------|
|
||||
| DB 连接超时(cpolar 隧道) | 加 connect_timeout + try/except 降级 |
|
||||
| 全表扫描性能 | 依赖 (site, waybill_no) 和 (site, scan_time) 索引 |
|
||||
| SF 运单数据量小(~1%) | 测试覆盖可能不足——需找有 SF 差缺的日期验证 |
|
||||
| `scan_time` 时区 | 统一用 `::date` cast,确认与服务器时区一致 |
|
||||
@@ -143,7 +143,7 @@ class CDP:
|
||||
"""绑定到单个页面目标的同步 CDP 客户端。"""
|
||||
|
||||
def __init__(self, ws_url):
|
||||
self.ws = websocket.create_connection(ws_url)
|
||||
self.ws = websocket.create_connection(ws_url, timeout=15) # 15s socket 超时:防 Electron 业务 tab 偶发不回包时 recv 无限阻塞
|
||||
self._id = 0
|
||||
|
||||
def call(self, method, **params):
|
||||
|
||||
@@ -87,7 +87,7 @@
|
||||
|
||||
| 接口 | 返回内容 | 是否含应到/已到计数 |
|
||||
|---|---|---|
|
||||
| `GET /status` | 各站 `login_state` + `expected/actual/undelivered_ready` + `business_date` + `worker_ready` | **不含**计数,只给就绪标志 |
|
||||
| `GET /status` | 各站 `login_state` + `expected/actual/undelivered_ready` + `business_date` + `worker_ready` + `ingest`(每站每类入库 ok/count/时间) | **不含**应到/已到件数计数(`ingest` 是入库条数,非核销件数) |
|
||||
| `GET /report` | `FileResponse(output/应到未到数据.xlsx)` | 计数只在 xlsx 里 |
|
||||
| `GET /data/{filename}` | 下载 `downloads/` 下某源文件 | 原始数据,非统计值 |
|
||||
|
||||
|
||||
@@ -10,23 +10,24 @@
|
||||
# 独占所有 page 操作;与主线程仅经 Queue + SQLite 通信。
|
||||
# 违反"路由不碰 Playwright"会崩(sync 对象跨线程访问)。
|
||||
#
|
||||
# 运行:python server.py (默认监听 0.0.0.0:8000)
|
||||
# 运行:python -m inbound_verify.cli.server (或 inbound-verify-server;默认监听 0.0.0.0:8000)
|
||||
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional
|
||||
|
||||
import uvicorn
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from inbound_verify.paths import DOWNLOAD_DIR, OUTPUT_DIR
|
||||
from inbound_verify.paths import OUTPUT_DIR
|
||||
from inbound_verify import state_store
|
||||
from inbound_verify.runtime import (
|
||||
HEARTBEAT_INTERVAL,
|
||||
@@ -35,6 +36,7 @@ from inbound_verify.runtime import (
|
||||
launch_and_prepare,
|
||||
run_heartbeat,
|
||||
)
|
||||
from inbound_verify import db_compare
|
||||
|
||||
# 全部站点;百世固定下载当天,不可配置偏移
|
||||
ALL_SITES = ["顺心", "百世", "中通", "韵达", "安能"]
|
||||
@@ -58,7 +60,7 @@ scheduler = BackgroundScheduler(daemon=True)
|
||||
def _worker_loop():
|
||||
"""worker 线程:启动 Playwright + 等就绪 + 任务循环(执行任务 + 心跳)。"""
|
||||
try:
|
||||
ctx = launch_and_prepare()
|
||||
ctx = launch_and_prepare(foreground=False)
|
||||
worker_state["ctx"] = ctx
|
||||
worker_state["ready"] = True
|
||||
# 【P1-2 重启自愈】worker 就绪后清理上轮遗留的 pending/running 僵尸任务
|
||||
@@ -98,46 +100,79 @@ def _worker_loop():
|
||||
print(">> [worker] 已退出。")
|
||||
|
||||
|
||||
def _enqueue_undelivered(site):
|
||||
"""定时 job:把该站 undelivered 任务投到队列(worker 串行处理;本线程不碰 Playwright)。"""
|
||||
def _in_active_window(active_start, active_end):
|
||||
"""当前本地时间是否落在激活时段内(避免半夜空跑)。
|
||||
- start/end 任一为空 → 不限时段(24h 活跃)
|
||||
- start == end(非空)→ 视为全天活跃
|
||||
- start < end → 半开区间 [start, end)
|
||||
- start > end → 跨午夜(如 22:00-06:00):now >= start 或 now < end
|
||||
"""
|
||||
if not active_start or not active_end:
|
||||
return True
|
||||
now = datetime.now().strftime("%H:%M")
|
||||
if active_start == active_end:
|
||||
return True
|
||||
if active_start < active_end:
|
||||
return active_start <= now < active_end
|
||||
return now >= active_start or now < active_end
|
||||
|
||||
|
||||
def _enqueue_fetch(site, kind):
|
||||
"""周期 job 回调:worker 就绪 + 在激活时段内 + 该(site,kind)无未完成任务时,投递一次抓取任务。
|
||||
跑在 APScheduler 线程池线程——绝不碰 Playwright,只经 SQLite + task_queue 通信。"""
|
||||
if not worker_state["ready"]:
|
||||
return # worker 未就绪:与 POST /tasks 的 409 同义,下个周期补抓
|
||||
cfg = state_store.get_fetch_schedule(site, kind)
|
||||
if not cfg or not cfg["enabled"]:
|
||||
return # 已禁用(job 本应已注销,双重保险)
|
||||
if not _in_active_window(cfg["active_start"], cfg["active_end"]):
|
||||
return # 不在激活时段,跳过本次 fire
|
||||
try:
|
||||
tid = state_store.create_task(site, "undelivered")
|
||||
task_queue.put((tid, {"site": site, "kind": "undelivered"}))
|
||||
print(f">> [定时] 投递 {site}/undelivered 任务 #{tid}")
|
||||
target_date = state_store.resolve_target_date(site, kind)
|
||||
tid = state_store.create_task_if_idle(
|
||||
site, kind, trigger="auto", target_date=target_date
|
||||
)
|
||||
if tid is None:
|
||||
return # 上一次同类任务还没跑完,跳过避免堆积
|
||||
task_queue.put((tid, {"site": site, "kind": kind}))
|
||||
print(f">> [周期] 投递 {site}/{kind} 任务 #{tid}")
|
||||
except Exception as e:
|
||||
print(f">> [定时] 投递 {site} 失败: {e}")
|
||||
print(f">> [周期] 投递 {site}/{kind} 失败: {e}")
|
||||
|
||||
|
||||
def _reschedule_site(site):
|
||||
"""按持久化配置(重新)注册或取消该站的每日定时 job。"""
|
||||
job_id = f"site_{site}"
|
||||
def _reschedule_fetch(site, kind):
|
||||
"""按持久化配置(重新)注册或取消该 (site,kind) 的周期抓取 job(IntervalTrigger)。"""
|
||||
job_id = f"{site}_{kind}"
|
||||
try:
|
||||
scheduler.remove_job(job_id)
|
||||
except Exception:
|
||||
pass
|
||||
cfg = state_store.get_all_config().get(site)
|
||||
if not cfg or not cfg.get("schedule_enabled") or not cfg.get("schedule_time"):
|
||||
return
|
||||
cfg = state_store.get_fetch_schedule(site, kind)
|
||||
if not cfg or not cfg["enabled"] or cfg["interval_minutes"] < 1:
|
||||
return # 未启用或频率非法:不注册(等同取消)
|
||||
try:
|
||||
hh, mm = cfg["schedule_time"].split(":")
|
||||
scheduler.add_job(
|
||||
_enqueue_undelivered,
|
||||
CronTrigger(hour=int(hh), minute=int(mm)),
|
||||
args=[site],
|
||||
_enqueue_fetch,
|
||||
IntervalTrigger(minutes=cfg["interval_minutes"]),
|
||||
args=[site, kind],
|
||||
id=job_id,
|
||||
replace_existing=True,
|
||||
)
|
||||
print(f">> [定时] 已注册 {site} 每日 {cfg['schedule_time']} 下载")
|
||||
print(
|
||||
f">> [周期] 已注册 {site}/{kind}:每 {cfg['interval_minutes']} 分钟"
|
||||
f"(激活 {cfg['active_start'] or '不限'}~{cfg['active_end'] or '不限'})"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f">> [定时] 注册 {site} 失败: {e}")
|
||||
print(f">> [周期] 注册 {site}/{kind} 失败: {e}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app):
|
||||
"""服务启停:起 worker 线程 / 通知 worker 停。"""
|
||||
state_store.init_db() # 先建表/迁移状态库,确保早于 worker 就绪的 /api/status 可用
|
||||
for site in ALL_SITES: # 按持久化配置注册各站每日定时 job
|
||||
_reschedule_site(site)
|
||||
for site in ALL_SITES: # 按持久化配置注册各站各 kind 的周期抓取 job
|
||||
for kind in state_store.allowed_kinds(site):
|
||||
_reschedule_fetch(site, kind)
|
||||
scheduler.start()
|
||||
print(">> [定时] 调度器已启动")
|
||||
t = threading.Thread(target=_worker_loop, daemon=True)
|
||||
@@ -156,11 +191,15 @@ app = FastAPI(title="InboundVerify 服务端", lifespan=lifespan)
|
||||
class TaskRequest(BaseModel):
|
||||
site: str
|
||||
kind: str
|
||||
force: bool = (
|
||||
False # 强制重下:忽略已落库去重,重新提交所有班次/交接单的导出任务(默认关)
|
||||
)
|
||||
date: Optional[str] = None # YYYY-MM-DD;指定则下载该日数据,否则走站点 offset
|
||||
|
||||
|
||||
@app.post("/tasks")
|
||||
def create_task(req: TaskRequest):
|
||||
"""提交任务 {site, kind} → 入队,返回 task_id。"""
|
||||
"""提交任务 {site, kind, force, date?} → 入队,返回 task_id。"""
|
||||
# 【P0】后端未就绪时直接拒绝,避免任务在 worker 启动前入队卡死
|
||||
if not worker_state["ready"]:
|
||||
raise HTTPException(
|
||||
@@ -168,8 +207,38 @@ def create_task(req: TaskRequest):
|
||||
)
|
||||
if (req.site, req.kind) not in TASK_HANDLERS:
|
||||
raise HTTPException(status_code=400, detail=f"无效任务: {req.site}/{req.kind}")
|
||||
task_id = state_store.create_task(req.site, req.kind)
|
||||
task_queue.put((task_id, {"site": req.site, "kind": req.kind}))
|
||||
# 指定日期合法性校验(仅在传了 date 时)
|
||||
if req.date:
|
||||
try:
|
||||
target_date = datetime.strptime(req.date, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"date 格式非法,需 YYYY-MM-DD: {req.date}"
|
||||
)
|
||||
today = datetime.now().date()
|
||||
if target_date > today:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"date 不可为未来日期: {req.date}"
|
||||
)
|
||||
if target_date < today - timedelta(days=31):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"date 超出 31 天回溯上限: {req.date}"
|
||||
)
|
||||
if req.site == "百世":
|
||||
raise HTTPException(
|
||||
status_code=400, detail="百世固定下载当天,不支持指定日期"
|
||||
)
|
||||
task_id = state_store.create_task(
|
||||
req.site,
|
||||
req.kind,
|
||||
trigger="manual",
|
||||
target_date=state_store.resolve_target_date(req.site, req.kind, req.date),
|
||||
force=req.force,
|
||||
)
|
||||
spec = {"site": req.site, "kind": req.kind, "force": req.force}
|
||||
if req.date:
|
||||
spec["date"] = req.date
|
||||
task_queue.put((task_id, spec))
|
||||
return {"task_id": task_id}
|
||||
|
||||
|
||||
@@ -186,30 +255,104 @@ def list_tasks(limit: int = 20):
|
||||
return state_store.list_tasks(limit)
|
||||
|
||||
|
||||
# ── DB 比对(基于 PostgreSQL,不依赖 Excel 文件)──
|
||||
|
||||
|
||||
class CompareRequest(BaseModel):
|
||||
site: str
|
||||
date: str # YYYY-MM-DD
|
||||
|
||||
|
||||
@app.post("/compare")
|
||||
def run_compare(req: CompareRequest):
|
||||
"""DB 差缺比对:以实到扫描日期为锚点,反推交接批次,展开全量比对。
|
||||
返回统计指标 + 差缺明细。
|
||||
"""
|
||||
# 合法性校验
|
||||
if req.site not in db_compare.SITE_COMPARE_CONFIG:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"不支持的站点: {req.site}(支持: {list(db_compare.SITE_COMPARE_CONFIG.keys())})",
|
||||
)
|
||||
try:
|
||||
target_date = datetime.strptime(req.date, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"date 格式非法,需 YYYY-MM-DD: {req.date}"
|
||||
)
|
||||
today = datetime.now().date()
|
||||
if target_date > today:
|
||||
raise HTTPException(status_code=400, detail=f"date 不可为未来日期: {req.date}")
|
||||
|
||||
result = db_compare.compare_site_date(req.site, req.date)
|
||||
if result is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"{req.site} {req.date}: 当天无实到数据,无法比对",
|
||||
)
|
||||
|
||||
return {
|
||||
"site": result.site,
|
||||
"date": result.date,
|
||||
"batches": result.batches,
|
||||
"stats": {
|
||||
"waybill_count": result.stats.waybill_count,
|
||||
"sf_wb_count": result.stats.sf_wb_count,
|
||||
"expected_pieces": result.stats.expected_pieces,
|
||||
"arrived_pieces": result.stats.arrived_pieces,
|
||||
"undelivered_pieces": result.stats.undelivered_pieces,
|
||||
"undelivered_wb": result.stats.undelivered_wb,
|
||||
"full_miss": result.stats.full_miss,
|
||||
"part_miss": result.stats.part_miss,
|
||||
"sf_undelivered": result.stats.sf_undelivered,
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"handover_no": r.handover_no,
|
||||
"waybill_no": r.waybill_no,
|
||||
"total_pieces": r.total_pieces,
|
||||
"arrived_pieces": r.arrived_pieces,
|
||||
"arrived_list": r.arrived_list,
|
||||
"is_sf": r.is_sf,
|
||||
}
|
||||
for r in result.rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@app.get("/status")
|
||||
def get_status():
|
||||
"""各站登录态 + 数据态(前端状态盘用),另含 worker 就绪状态。"""
|
||||
"""各站登录态 + 数据态 + 入库态(前端状态盘用),另含 worker 就绪状态。"""
|
||||
return {
|
||||
"worker_ready": worker_state["ready"],
|
||||
"worker_error": worker_state["error"],
|
||||
"sites": state_store.get_all_status(),
|
||||
"ingest": state_store.get_all_ingest_state(),
|
||||
}
|
||||
|
||||
|
||||
class FetchScheduleSpec(BaseModel):
|
||||
enabled: bool
|
||||
active_start: str = ""
|
||||
active_end: str = ""
|
||||
interval_minutes: int
|
||||
|
||||
|
||||
class ConfigRequest(BaseModel):
|
||||
expected_offset: Optional[int] = None
|
||||
actual_offset: Optional[int] = None
|
||||
# DEPRECATED:旧"每日定点"字段,保留一个发布周期兼容旧前端请求(收到即忽略)
|
||||
schedule_enabled: Optional[bool] = None
|
||||
schedule_time: Optional[str] = None
|
||||
settings: Optional[Dict[str, str]] = None
|
||||
fetch_schedules: Optional[Dict[str, FetchScheduleSpec]] = None
|
||||
|
||||
|
||||
def _default_cfg():
|
||||
return {
|
||||
"expected_offset": 0,
|
||||
"actual_offset": 0,
|
||||
"schedule_enabled": False,
|
||||
"schedule_time": "",
|
||||
"fetch_schedules": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -224,7 +367,6 @@ def get_config():
|
||||
def set_config(site: str, req: ConfigRequest):
|
||||
if site not in ALL_SITES:
|
||||
raise HTTPException(status_code=400, detail=f"未知站点: {site}")
|
||||
cur = state_store.get_all_config().get(site, _default_cfg())
|
||||
# 应到/实到偏移(百世锁定当天)
|
||||
for kind, val in (
|
||||
("expected", req.expected_offset),
|
||||
@@ -236,18 +378,25 @@ def set_config(site: str, req: ConfigRequest):
|
||||
status_code=400, detail="百世固定下载当天,不可配置偏移"
|
||||
)
|
||||
state_store.set_offset(site, kind, val)
|
||||
# 每日定时(所有站可配,含百世)
|
||||
if req.schedule_enabled is not None or req.schedule_time is not None:
|
||||
enabled = (
|
||||
req.schedule_enabled
|
||||
if req.schedule_enabled is not None
|
||||
else cur["schedule_enabled"]
|
||||
)
|
||||
time_str = (
|
||||
req.schedule_time if req.schedule_time is not None else cur["schedule_time"]
|
||||
)
|
||||
state_store.set_schedule(site, enabled, time_str)
|
||||
_reschedule_site(site)
|
||||
# 周期抓取调度(应到/实到各自独立;百世只允许 undelivered)
|
||||
if req.fetch_schedules:
|
||||
allowed = state_store.allowed_kinds(site)
|
||||
for kind, spec in req.fetch_schedules.items():
|
||||
if kind not in allowed:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"站点 {site} 不支持抓取类型 {kind}(允许: {list(allowed)})",
|
||||
)
|
||||
interval = max(1, min(1440, int(spec.interval_minutes)))
|
||||
state_store.set_fetch_schedule(
|
||||
site,
|
||||
kind,
|
||||
spec.enabled,
|
||||
spec.active_start,
|
||||
spec.active_end,
|
||||
interval,
|
||||
)
|
||||
_reschedule_fetch(site, kind)
|
||||
# 站点专属配置(密码/账号/路径…)
|
||||
if req.settings:
|
||||
for k, v in req.settings.items():
|
||||
@@ -277,20 +426,6 @@ def download_report():
|
||||
return FileResponse(path, filename=REPORT_FILE)
|
||||
|
||||
|
||||
@app.get("/data/{filename}")
|
||||
def download_data(filename: str):
|
||||
"""下载 downloads/ 下的数据文件(防路径穿越)。"""
|
||||
if not filename or "/" in filename or "\\" in filename or ".." in filename:
|
||||
raise HTTPException(status_code=400, detail="非法文件名")
|
||||
path = os.path.join(DOWNLOAD_DIR, filename)
|
||||
# 双重校验:解析后绝对路径仍在 DOWNLOAD_DIR 内
|
||||
if not os.path.abspath(path).startswith(os.path.abspath(DOWNLOAD_DIR) + os.sep):
|
||||
raise HTTPException(status_code=400, detail="非法路径")
|
||||
if not os.path.isfile(path):
|
||||
raise HTTPException(status_code=404, detail="文件不存在")
|
||||
return FileResponse(path, filename=filename)
|
||||
|
||||
|
||||
def main():
|
||||
"""服务模式入口。传字符串导入路径(规范写法;不开 reload/workers 时进程内 import,行为等价)。"""
|
||||
uvicorn.run("inbound_verify.cli.server:app", host="0.0.0.0", port=8000)
|
||||
|
||||
@@ -75,6 +75,12 @@ def process(name):
|
||||
df_exp = pd.read_excel(exp_path, dtype=str).fillna("")
|
||||
df_act = pd.read_excel(act_path, dtype=str).fillna("")
|
||||
|
||||
if name == "韵达":
|
||||
# 韵达实到数据有重复行(同子单号出现两次),保留交接单号为空的(到/接件扫描),
|
||||
# 丢弃交接单号不为空的(派件/签收等),再按子单号去重。
|
||||
df_act = df_act[df_act["交接单号"].astype(str).str.strip() == ""]
|
||||
df_act = df_act.drop_duplicates(subset=["子单号"], keep="last")
|
||||
|
||||
# 同一运单可能有多条交接记录,按运单号去重、保留首条
|
||||
dup = int(df_exp[cfg["exp_wb"]].duplicated().sum())
|
||||
df_exp = df_exp.drop_duplicates(subset=[cfg["exp_wb"]], keep="first")
|
||||
|
||||
658
inbound_verify/db_compare.py
Normal file
658
inbound_verify/db_compare.py
Normal file
@@ -0,0 +1,658 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
db_compare.py — 基于 PostgreSQL 的应到未到差缺比对引擎。
|
||||
|
||||
与 compare.py(Excel 版)并行:本模块直接从 DB 查询数据进行比对,
|
||||
不依赖 downloads/ 下的 Excel 文件。
|
||||
|
||||
核心思路:以实到扫描日期为锚点 → 反推交接批次 → 展开批次全量比对。
|
||||
|
||||
每个站点只需提供配置(waybill 列名 / piece 列名 / 是否有 SF 特殊处理),
|
||||
核心比对逻辑完全通用。
|
||||
|
||||
顺心站点 SF 运单特殊处理:SF 运单的子单号(piece_no)为随机号码,不能用
|
||||
COUNT(DISTINCT piece_no) 去重计数,改为 COUNT(*) 行计数。
|
||||
|
||||
用法:
|
||||
from inbound_verify.db_compare import compare_site_date, SITE_COMPARE_CONFIG
|
||||
|
||||
result = compare_site_date("顺心", "2026-07-25")
|
||||
if result:
|
||||
print(result.stats)
|
||||
for row in result.rows:
|
||||
print(row)
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
import psycopg
|
||||
import yaml
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side
|
||||
|
||||
from inbound_verify.paths import CONFIG_PATH, OUTPUT_DIR, DOWNLOAD_DIR
|
||||
from inbound_verify.domain import _site_cfg, ALL_REPORT_SITES, BAISHI_COLUMNS
|
||||
|
||||
# ============================== 结果类型 ==============================
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareStats:
|
||||
"""单站点/单批次比对统计。"""
|
||||
|
||||
waybill_count: int = 0 # 应到运单数
|
||||
expected_pieces: int = 0 # 应到件数
|
||||
arrived_pieces: int = 0 # 实到件数
|
||||
undelivered_pieces: int = 0 # 未到件数
|
||||
undelivered_wb: int = 0 # 差缺运单数
|
||||
full_miss: int = 0 # 完全未到
|
||||
part_miss: int = 0 # 部分未到
|
||||
sf_wb_count: int = 0 # SF 运单数
|
||||
sf_undelivered: int = 0 # SF 差缺数
|
||||
|
||||
|
||||
@dataclass
|
||||
class UndeliveredRow:
|
||||
"""单条差缺明细。"""
|
||||
|
||||
handover_no: str = "" # 交接单号
|
||||
waybill_no: str = "" # 运单号
|
||||
total_pieces: int = 0 # 总件数(交接件数)
|
||||
arrived_pieces: int = 0 # 已到件数
|
||||
arrived_list: list = field(default_factory=list) # 已到单号列表
|
||||
is_sf: bool = False # 是否 SF 运单
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareResult:
|
||||
"""一次比对的完整结果。"""
|
||||
|
||||
site: str = ""
|
||||
date: str = ""
|
||||
batches: list = field(default_factory=list) # 涉及的交接批次
|
||||
stats: CompareStats = field(default_factory=CompareStats)
|
||||
rows: list = field(default_factory=list) # UndeliveredRow 列表
|
||||
|
||||
|
||||
# ============================== 站点比对配置 ==============================
|
||||
|
||||
|
||||
@dataclass
|
||||
class SiteCompareConfig:
|
||||
"""DB 比对的站点参数。"""
|
||||
|
||||
name: str # 站点名
|
||||
has_sf: bool = False # 是否需要区分 SF 运单
|
||||
|
||||
|
||||
# 四站点 DB 比对配置(百世不参与 4 站比对)
|
||||
SITE_COMPARE_CONFIG: dict[str, SiteCompareConfig] = {
|
||||
"顺心": SiteCompareConfig(name="顺心", has_sf=True),
|
||||
"中通": SiteCompareConfig(name="中通", has_sf=False),
|
||||
"韵达": SiteCompareConfig(name="韵达", has_sf=False),
|
||||
"安能": SiteCompareConfig(name="安能", has_sf=False),
|
||||
}
|
||||
|
||||
|
||||
# ============================== DB 连接 ==============================
|
||||
|
||||
|
||||
def _load_pg_config():
|
||||
"""从 config.yaml 读 postgres 段。与 store.py 共用同一配置源。"""
|
||||
if not os.path.exists(CONFIG_PATH):
|
||||
raise FileNotFoundError(
|
||||
f"未找到配置文件 {CONFIG_PATH}(请参考 config.example.yaml 创建 config.yaml)"
|
||||
)
|
||||
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
|
||||
cfg = yaml.safe_load(f) or {}
|
||||
pg = cfg.get("postgres") or {}
|
||||
return {
|
||||
"host": pg.get("host", "127.0.0.1"),
|
||||
"port": int(pg.get("port", 5432)),
|
||||
"user": pg.get("user", "postgres"),
|
||||
"password": pg.get("password", ""),
|
||||
"dbname": pg.get("dbname", "CQHXDB"),
|
||||
"schema": pg.get("schema", "inbound_verify"),
|
||||
"connect_timeout_seconds": int(pg.get("connect_timeout_seconds", 5)),
|
||||
}
|
||||
|
||||
|
||||
def _connect():
|
||||
c = _load_pg_config()
|
||||
return psycopg.connect(
|
||||
host=c["host"],
|
||||
port=c["port"],
|
||||
dbname=c["dbname"],
|
||||
user=c["user"],
|
||||
password=c["password"],
|
||||
options=f"-c search_path={c['schema']} -c statement_timeout=30s",
|
||||
connect_timeout=c["connect_timeout_seconds"],
|
||||
)
|
||||
|
||||
|
||||
# ============================== 核心比对逻辑 ==============================
|
||||
|
||||
|
||||
def compare_site_date(site: str, target_date: str) -> CompareResult | None:
|
||||
"""对指定站点和日期执行 DB 差缺比对。
|
||||
|
||||
算法:
|
||||
1. 取 scan_time::date = target_date 的实到运单(锚点)
|
||||
2. 反推这些运单所属的交接批次(handover_no)
|
||||
3. 展开批次全量应到运单
|
||||
4. 查询批次全量实到扫描
|
||||
5. 逐运单比对差缺(SF/non-SF 分支处理)
|
||||
|
||||
Args:
|
||||
site: 站点名("顺心"/"中通"/"韵达"/"安能")
|
||||
target_date: 日期 "YYYY-MM-DD"
|
||||
|
||||
Returns:
|
||||
CompareResult 或 None(当天无实到数据时返回 None)
|
||||
"""
|
||||
cfg = SITE_COMPARE_CONFIG.get(site)
|
||||
if cfg is None:
|
||||
print(f"[db_compare] 不支持的站点: {site}")
|
||||
return None
|
||||
|
||||
try:
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
|
||||
# ── Step 1: 取实到锚点 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT DISTINCT waybill_no FROM actual_record
|
||||
WHERE site = %s AND scan_time::date = %s
|
||||
""",
|
||||
(site, target_date),
|
||||
)
|
||||
anchor_wbs = [r[0] for r in cur.fetchall()]
|
||||
if not anchor_wbs:
|
||||
print(f"[db_compare] {site} {target_date}: 当天无实到数据")
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
# ── Step 2: 反推交接批次 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT DISTINCT e.handover_no FROM expected_record e
|
||||
WHERE e.site = %s AND e.waybill_no = ANY(%s)
|
||||
""",
|
||||
(site, anchor_wbs),
|
||||
)
|
||||
batches = [r[0] for r in cur.fetchall()]
|
||||
|
||||
# ── Step 3: 展开批次全量应到 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, handover_no, handover_pieces
|
||||
FROM expected_record
|
||||
WHERE site = %s AND handover_no = ANY(%s)
|
||||
ORDER BY handover_no, waybill_no
|
||||
""",
|
||||
(site, batches),
|
||||
)
|
||||
exp_rows = cur.fetchall() # [(waybill_no, handover_no, handover_pieces), ...]
|
||||
|
||||
if not exp_rows:
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
all_wbs = [r[0] for r in exp_rows]
|
||||
|
||||
# ── Step 4: 取批次全量实到 ──
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, piece_no FROM actual_record
|
||||
WHERE site = %s AND waybill_no = ANY(%s)
|
||||
ORDER BY waybill_no, piece_no
|
||||
""",
|
||||
(site, all_wbs),
|
||||
)
|
||||
act_rows = cur.fetchall() # [(waybill_no, piece_no), ...]
|
||||
|
||||
conn.close()
|
||||
|
||||
# ── Step 5: 逐运单比对 ──
|
||||
return _do_compare(site, target_date, batches, exp_rows, act_rows, cfg)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[db_compare] {site} {target_date} 比对异常: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def compare_site_batch(site: str, handover_no: str) -> CompareResult | None:
|
||||
"""按指定交接单号执行全批次比对(不依赖实到锚点)。
|
||||
|
||||
用于已知交接单号后精确比对某一批次。
|
||||
"""
|
||||
cfg = SITE_COMPARE_CONFIG.get(site)
|
||||
if cfg is None:
|
||||
print(f"[db_compare] 不支持的站点: {site}")
|
||||
return None
|
||||
|
||||
try:
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, handover_no, handover_pieces
|
||||
FROM expected_record
|
||||
WHERE site = %s AND handover_no = %s
|
||||
ORDER BY waybill_no
|
||||
""",
|
||||
(site, handover_no),
|
||||
)
|
||||
exp_rows = cur.fetchall()
|
||||
if not exp_rows:
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
all_wbs = [r[0] for r in exp_rows]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT waybill_no, piece_no FROM actual_record
|
||||
WHERE site = %s AND waybill_no = ANY(%s)
|
||||
ORDER BY waybill_no, piece_no
|
||||
""",
|
||||
(site, all_wbs),
|
||||
)
|
||||
act_rows = cur.fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
return _do_compare(
|
||||
site,
|
||||
f"batch:{handover_no}",
|
||||
[handover_no],
|
||||
exp_rows,
|
||||
act_rows,
|
||||
cfg,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[db_compare] {site} batch:{handover_no} 比对异常: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ============================== 比对核心 ==============================
|
||||
|
||||
|
||||
def _do_compare(
|
||||
site: str,
|
||||
label: str,
|
||||
batches: list[str],
|
||||
exp_rows: list[tuple], # [(waybill_no, handover_no, handover_pieces), ...]
|
||||
act_rows: list[tuple], # [(waybill_no, piece_no), ...]
|
||||
cfg: SiteCompareConfig,
|
||||
) -> CompareResult:
|
||||
"""执行逐运单比对,产出统计 + 差缺明细。
|
||||
|
||||
与 compare.py:process() 口径一致:
|
||||
- 应到件数 = handover_pieces(交接件数)
|
||||
- 实到件数 = SF ? COUNT(*) : COUNT(DISTINCT piece_no)
|
||||
- arrived_cnt >= handover_pieces → 足额到货,跳过
|
||||
"""
|
||||
# 构建实到索引: waybill_no → [piece_no, ...](保留所有行,不去重)
|
||||
act_by_wb: dict[str, list[str]] = {}
|
||||
for wb, piece in act_rows:
|
||||
act_by_wb.setdefault(wb, []).append(piece)
|
||||
|
||||
stats = CompareStats()
|
||||
rows: list[UndeliveredRow] = []
|
||||
max_arrived = 0
|
||||
|
||||
for wb, handover_no, handover_pcs in exp_rows:
|
||||
handover_pcs = handover_pcs or 0
|
||||
if handover_pcs <= 0:
|
||||
continue
|
||||
|
||||
stats.waybill_count += 1
|
||||
stats.expected_pieces += handover_pcs
|
||||
|
||||
is_sf = cfg.has_sf and wb.startswith("SF")
|
||||
if is_sf:
|
||||
stats.sf_wb_count += 1
|
||||
|
||||
all_pieces = act_by_wb.get(wb, [])
|
||||
|
||||
if is_sf:
|
||||
# SF: 行计数,不去重(piece_no 是随机号码)
|
||||
arrived_cnt = len(all_pieces)
|
||||
arrived_list = list(all_pieces)
|
||||
else:
|
||||
# non-SF: 子单号去重
|
||||
unique_pieces = list(dict.fromkeys(all_pieces)) # 保序去重
|
||||
arrived_cnt = len(unique_pieces)
|
||||
arrived_list = unique_pieces
|
||||
|
||||
stats.arrived_pieces += arrived_cnt
|
||||
|
||||
if arrived_cnt >= handover_pcs:
|
||||
continue # 足额或溢到,不进差缺表
|
||||
|
||||
if arrived_cnt == 0:
|
||||
stats.full_miss += 1
|
||||
else:
|
||||
stats.part_miss += 1
|
||||
|
||||
if is_sf:
|
||||
stats.sf_undelivered += 1
|
||||
|
||||
max_arrived = max(max_arrived, arrived_cnt)
|
||||
rows.append(
|
||||
UndeliveredRow(
|
||||
handover_no=handover_no,
|
||||
waybill_no=wb,
|
||||
total_pieces=handover_pcs,
|
||||
arrived_pieces=arrived_cnt,
|
||||
arrived_list=arrived_list,
|
||||
is_sf=is_sf,
|
||||
)
|
||||
)
|
||||
|
||||
stats.undelivered_pieces = max(0, stats.expected_pieces - stats.arrived_pieces)
|
||||
stats.undelivered_wb = stats.full_miss + stats.part_miss
|
||||
|
||||
result = CompareResult(
|
||||
site=site,
|
||||
date=label,
|
||||
batches=batches,
|
||||
stats=stats,
|
||||
rows=rows,
|
||||
)
|
||||
|
||||
# 打印摘要
|
||||
print(
|
||||
f"[db_compare] {site} {label}: "
|
||||
f"batches={len(batches)}, "
|
||||
f"wb={stats.waybill_count}(SF:{stats.sf_wb_count}), "
|
||||
f"exp={stats.expected_pieces}, arr={stats.arrived_pieces}, "
|
||||
f"miss={stats.undelivered_pieces}, "
|
||||
f"miss_wb={stats.undelivered_wb}(full={stats.full_miss}, part={stats.part_miss})"
|
||||
)
|
||||
if stats.sf_undelivered:
|
||||
print(f" SF 差缺: {stats.sf_undelivered} 个运单")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ============================== Excel 输出 ==============================
|
||||
|
||||
|
||||
# 样式常量(与 compare.py 对齐)
|
||||
_FONT = "微软雅黑"
|
||||
_BLUE = "305496"
|
||||
|
||||
_HEADER_FILL = PatternFill("solid", fgColor=_BLUE)
|
||||
_HEADER_FONT = Font(name=_FONT, bold=True, color="FFFFFF", size=11)
|
||||
_BODY_FONT = Font(name=_FONT, size=10)
|
||||
_THIN = Side(style="thin", color="D9D9D9")
|
||||
_BORDER = Border(left=_THIN, right=_THIN, top=_THIN, bottom=_THIN)
|
||||
|
||||
|
||||
def write_result_excel(result: CompareResult, output_path: str | None = None) -> str:
|
||||
"""将比对结果写入 Excel 文件。
|
||||
|
||||
Args:
|
||||
result: compare_site_date 或 compare_site_batch 的返回值
|
||||
output_path: 输出路径,为 None 时自动生成:
|
||||
output/{站}-{日期}-未到数据.xlsx
|
||||
|
||||
Returns:
|
||||
实际写入的文件路径
|
||||
"""
|
||||
if output_path is None:
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
date_tag = result.date.replace(":", "-").replace("batch:", "batch-")
|
||||
output_path = os.path.join(
|
||||
OUTPUT_DIR, f"{result.site}-{date_tag}-未到数据.xlsx"
|
||||
)
|
||||
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = result.site
|
||||
|
||||
_write_sheet(ws, result)
|
||||
wb.save(output_path)
|
||||
print(f"[db_compare] Excel 已输出: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
def _write_sheet(ws, result: CompareResult):
|
||||
"""写单个站点的差缺明细 sheet。"""
|
||||
s = result.stats
|
||||
rows = result.rows
|
||||
|
||||
# 动态列: 交接单号 | 运单号 | 总件数 | 已到单号1 | 已到单号2 | ...
|
||||
max_arrived = max((len(r.arrived_list) for r in rows), default=0)
|
||||
columns = ["交接单号", "运单号", "总件数"] + [
|
||||
f"已到单号{i + 1}" for i in range(max_arrived)
|
||||
]
|
||||
|
||||
ws.sheet_view.showGridLines = False
|
||||
|
||||
# 表头
|
||||
ws.append(columns)
|
||||
for c in range(1, len(columns) + 1):
|
||||
cell = ws.cell(row=1, column=c)
|
||||
cell.fill = _HEADER_FILL
|
||||
cell.font = _HEADER_FONT
|
||||
cell.alignment = Alignment(horizontal="center", vertical="center")
|
||||
cell.border = _BORDER
|
||||
|
||||
# 数据行
|
||||
for row in rows:
|
||||
values = {
|
||||
"交接单号": row.handover_no,
|
||||
"运单号": row.waybill_no,
|
||||
"总件数": row.total_pieces,
|
||||
}
|
||||
for i, piece in enumerate(row.arrived_list):
|
||||
values[f"已到单号{i + 1}"] = piece
|
||||
ws.append([values.get(c, "") for c in columns])
|
||||
|
||||
# 格式
|
||||
for r in range(2, ws.max_row + 1):
|
||||
for c, col in enumerate(columns, start=1):
|
||||
cell = ws.cell(row=r, column=c)
|
||||
cell.font = _BODY_FONT
|
||||
cell.border = _BORDER
|
||||
if col == "总件数":
|
||||
cell.number_format = "#,##0"
|
||||
cell.alignment = Alignment(horizontal="right", vertical="center")
|
||||
else:
|
||||
cell.number_format = "@"
|
||||
|
||||
# 列宽
|
||||
for c, col in enumerate(columns, start=1):
|
||||
body_lens = [
|
||||
len(str(ws.cell(row=r, column=c).value or ""))
|
||||
for r in range(2, ws.max_row + 1)
|
||||
]
|
||||
width = min(max([len(str(col))] + body_lens) + 4, 36)
|
||||
ws.column_dimensions[ws.cell(row=1, column=c).column_letter].width = max(
|
||||
width, 12
|
||||
)
|
||||
|
||||
ws.freeze_panes = "A2"
|
||||
|
||||
|
||||
# ============================== 全站汇总报表(DB 版)=============================
|
||||
|
||||
|
||||
def _stats_to_dict(s: CompareStats) -> dict:
|
||||
"""CompareStats -> build_summary 要的中文键 stats dict。"""
|
||||
return {
|
||||
"运单数": s.waybill_count,
|
||||
"应到件": s.expected_pieces,
|
||||
"已到件": s.arrived_pieces,
|
||||
"未到件": s.undelivered_pieces,
|
||||
"完全未到": s.full_miss,
|
||||
"部分未到": s.part_miss,
|
||||
}
|
||||
|
||||
|
||||
def _target_date_for(site: str) -> str:
|
||||
"""4 站比对锚点:today - actual_offset(以实到扫描日为锚,与 _site_undelivered_handler 一致)。"""
|
||||
from inbound_verify import state_store # 懒导入,避免成环
|
||||
|
||||
offset = state_store.get_offset(site, "actual")
|
||||
return (date.today() - timedelta(days=offset)).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _baishi_from_pg(cur, target: str):
|
||||
"""查百世当日基数(baishi_daily_stats)+ 当天未到明细(undelivered_record 按 ingested_at 过滤)。
|
||||
返回 (stats_dict_or_None, rows_or_None);基数与明细均无 → (None, None)。
|
||||
|
||||
undelivered_record 是 UPSERT 累积表;按 ingested_at::date = target 取当天入库的未到快照
|
||||
(= 当天下载的当前未到,站点已剔除已到),避免累积偏大。
|
||||
"""
|
||||
cur.execute(
|
||||
"SELECT expected_pieces, arrived_pieces, undelivered_pieces "
|
||||
"FROM baishi_daily_stats WHERE site = %s AND business_date = %s",
|
||||
("百世", target),
|
||||
)
|
||||
basis = cur.fetchone()
|
||||
cur.execute(
|
||||
"SELECT waybill_no, piece_no, biz_type, last_scan FROM undelivered_record "
|
||||
"WHERE site = %s AND ingested_at::date = %s",
|
||||
("百世", target),
|
||||
)
|
||||
detail = cur.fetchall()
|
||||
if basis is None and not detail:
|
||||
return (None, None)
|
||||
exp = basis[0] if basis else None
|
||||
arr = basis[1] if basis else None
|
||||
# 未到件优先取基数差(baishi_daily_stats.undelivered_pieces,与应到/已到同源自洽);
|
||||
# 基数缺失时退回明细行数。
|
||||
undel = basis[2] if (basis and basis[2] is not None) else len(detail)
|
||||
wb_count = len({r[0] for r in detail if r[0]}) # 运单号去重
|
||||
rows = [
|
||||
{
|
||||
"类型": r[2] or "",
|
||||
"子单号": r[1] or "",
|
||||
"运单号": r[0] or "",
|
||||
"最新扫描记录": r[3] or "",
|
||||
}
|
||||
for r in detail
|
||||
]
|
||||
stats = {
|
||||
"运单数": wb_count,
|
||||
"应到件": exp,
|
||||
"已到件": arr,
|
||||
"未到件": undel,
|
||||
"完全未到": None,
|
||||
"部分未到": None,
|
||||
}
|
||||
return (stats, rows)
|
||||
|
||||
|
||||
def build_full_report(date=None) -> str:
|
||||
"""DB 版全站汇总报表:4 站走 DB 比对、百世走 PG,复用 compare.build_summary 渲染。
|
||||
产出 output/应到未到数据.xlsx(/report 下载)。date=None 时各站按 actual_offset 算锚点(以实到扫描日为锚)。
|
||||
返回输出路径。"""
|
||||
from inbound_verify import compare # 复用 build_summary / write_station / OUTFILE
|
||||
|
||||
print("[db_compare] 开始生成全站汇总报表 ...")
|
||||
wb = Workbook()
|
||||
wb.remove(wb.active)
|
||||
summary_ws = wb.create_sheet("汇总报表")
|
||||
|
||||
results = [] # [(name, stats_dict_or_None)],顺序 ALL_REPORT_SITES
|
||||
site_targets = {} # name -> target_date(汇总表"数据日期"列)
|
||||
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
for name in ALL_REPORT_SITES:
|
||||
if name == "百世":
|
||||
target = date or datetime.now().strftime("%Y-%m-%d")
|
||||
site_targets[name] = target
|
||||
stats, rows = _baishi_from_pg(cur, target)
|
||||
results.append((name, stats))
|
||||
if rows is not None:
|
||||
compare.write_station(wb.create_sheet(name), BAISHI_COLUMNS, rows)
|
||||
continue
|
||||
if name not in SITE_COMPARE_CONFIG:
|
||||
results.append((name, None))
|
||||
continue
|
||||
target = date or _target_date_for(name)
|
||||
site_targets[name] = target
|
||||
result = compare_site_date(name, target)
|
||||
if result is not None:
|
||||
results.append((name, _stats_to_dict(result.stats)))
|
||||
_write_sheet(wb.create_sheet(name), result)
|
||||
else:
|
||||
results.append((name, None))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
compare.build_summary(
|
||||
summary_ws,
|
||||
results,
|
||||
datetime.now().strftime("%Y-%m-%d %H:%M"),
|
||||
dates=site_targets,
|
||||
)
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
wb.save(compare.OUTFILE)
|
||||
print(f"[db_compare] 全站汇总已输出: {compare.OUTFILE}")
|
||||
for name, s in results:
|
||||
print(f" {name}:未到 {s['未到件']} 件" if s else f" {name}:无数据,跳过")
|
||||
return compare.OUTFILE
|
||||
|
||||
|
||||
# ============================== 终端验证入口 ==============================
|
||||
|
||||
|
||||
def main():
|
||||
"""命令行验证入口:
|
||||
python -m inbound_verify.db_compare 顺心 2026-07-25
|
||||
"""
|
||||
import sys
|
||||
|
||||
site = sys.argv[1] if len(sys.argv) > 1 else "顺心"
|
||||
target_date = sys.argv[2] if len(sys.argv) > 2 else "2026-07-25"
|
||||
|
||||
result = compare_site_date(site, target_date)
|
||||
if result is None:
|
||||
print(f"{site} {target_date}: 无结果")
|
||||
return
|
||||
|
||||
print(f"\n=== {result.site} {result.date} 差缺明细 ===")
|
||||
print(f"涉及批次: {result.batches}")
|
||||
print(f"应到运单: {result.stats.waybill_count} (SF: {result.stats.sf_wb_count})")
|
||||
print(f"应到件数: {result.stats.expected_pieces}")
|
||||
print(f"实到件数: {result.stats.arrived_pieces}")
|
||||
print(f"未到件数: {result.stats.undelivered_pieces}")
|
||||
print(
|
||||
f"差缺运单: {result.stats.undelivered_wb} (完全未到: {result.stats.full_miss}, 部分未到: {result.stats.part_miss})"
|
||||
)
|
||||
if result.stats.sf_undelivered:
|
||||
print(f"SF 差缺: {result.stats.sf_undelivered}")
|
||||
|
||||
if result.rows:
|
||||
print(f"\n--- 差缺明细 (共 {len(result.rows)} 条) ---")
|
||||
for row in result.rows[:20]:
|
||||
sf = "[SF]" if row.is_sf else ""
|
||||
arrived_preview = row.arrived_list[:5]
|
||||
print(
|
||||
f" {sf} {row.waybill_no}: "
|
||||
f"应到{row.total_pieces}件, 实到{row.arrived_pieces}件"
|
||||
f" {f'已到: {arrived_preview}' if arrived_preview else ''}"
|
||||
)
|
||||
if len(result.rows) > 20:
|
||||
print(f" ... 还有 {len(result.rows) - 20} 条")
|
||||
|
||||
# 输出 Excel
|
||||
path = write_result_excel(result)
|
||||
print(f"\n结果文件: {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -18,3 +18,6 @@ CONFIG_PATH = os.path.join(BASE_DIR, "config.yaml")
|
||||
|
||||
# 状态存储(SQLite,阶段0:心跳 / 登录态 / 数据态持久化,重启不丢)
|
||||
STATE_DB_PATH = os.path.join(BASE_DIR, "state", "state.db")
|
||||
|
||||
# 错误截图目录(下载流程失败时自动截取,供问题排查)
|
||||
SCREENSHOT_DIR = os.path.join(BASE_DIR, "logs", "screenshots")
|
||||
|
||||
@@ -16,13 +16,12 @@ import socket
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
import yaml
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
from inbound_verify.paths import DOWNLOAD_DIR, CONFIG_PATH
|
||||
from inbound_verify.domain import SITE_UNDELIVERED_FILE
|
||||
from inbound_verify.paths import CONFIG_PATH, SCREENSHOT_DIR
|
||||
|
||||
from inbound_verify import state_store
|
||||
from inbound_verify.sites import shunxin, baishi, zto, yunda, anneng
|
||||
@@ -50,31 +49,6 @@ APP_SITES = {"安能"}
|
||||
# 心跳间隔(秒)
|
||||
HEARTBEAT_INTERVAL = 30
|
||||
|
||||
# 各站最终数据文件名(探测"数据是否已跑出来");百世为单流程
|
||||
DATA_FILENAMES = {
|
||||
"顺心": {
|
||||
"expected": "顺心-应到货物数据.xlsx",
|
||||
"actual": "顺心-实到货物数据.xlsx",
|
||||
"undelivered": "顺心-未到数据.xlsx",
|
||||
},
|
||||
"中通": {
|
||||
"expected": "中通-应到货物数据.xlsx",
|
||||
"actual": "中通-实到货物数据.xlsx",
|
||||
"undelivered": "中通-未到数据.xlsx",
|
||||
},
|
||||
"韵达": {
|
||||
"expected": "韵达-应到货物数据.xlsx",
|
||||
"actual": "韵达-实到货物数据.xlsx",
|
||||
"undelivered": "韵达-未到数据.xlsx",
|
||||
},
|
||||
"安能": {
|
||||
"expected": "安能-应到货物数据.xlsx",
|
||||
"actual": "安能-实到货物数据.xlsx",
|
||||
"undelivered": "安能-未到数据.xlsx",
|
||||
},
|
||||
"百世": {"expected": "", "actual": "", "undelivered": "百世-应到未到货物数据.xlsx"},
|
||||
}
|
||||
|
||||
|
||||
# ============================ 安能启动(CDP)============================
|
||||
|
||||
@@ -102,13 +76,37 @@ def _wait_cdp_up(port, timeout=60.0):
|
||||
return False
|
||||
|
||||
|
||||
# 【环境兼容】宿主 shell(Codex/VS Code 插件、WorkBuddy 等)会向子进程注入一批与业务
|
||||
# 无关的变量,实测会让安能应用登录后反复弹出“获取试用网点接口报错”:
|
||||
# - HTTP(S)_PROXY=http://127.0.0.1:8800(QuickQ 加速器代理):安能的 wnp.ane56.com
|
||||
# 接口请求被塞进第三方代理后返回 400/用户未登录;
|
||||
# - VSCODE_* / CODEX_* / EFC_*:VS Code 扩展宿主注入(IPC、PID、NLS、ESM 等);
|
||||
# - NODE_TLS_REJECT_UNAUTHORIZED / DEBUG / RUST_LOG 等宿主调试变量。
|
||||
# 另:ELECTRON_RUN_AS_NODE=1 会把安能当作纯 Node 运行(拒绝 Chromium 参数、启动即
|
||||
# 退出 rc=9);NODE_OPTIONS 同样会干扰。拉起前全部摘掉,尽量还原终端手动启动环境。
|
||||
_ANNENG_STRIP_PREFIXES = ("VSCODE_", "CODEX_", "EFC_")
|
||||
_ANNENG_STRIP_EXACT = {
|
||||
"NODE_OPTIONS",
|
||||
"ELECTRON_RUN_AS_NODE",
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"ALL_PROXY",
|
||||
"NO_PROXY",
|
||||
"NODE_TLS_REJECT_UNAUTHORIZED",
|
||||
"NODEFAULTCURRENTDIRECTORYINEXEPATH",
|
||||
"DEBUG",
|
||||
"RUST_LOG",
|
||||
"APPLICATION_INSIGHTS_NO_STATSBEAT",
|
||||
}
|
||||
|
||||
|
||||
def launch_anneng(app_path):
|
||||
"""以调试模式启动安能 Electron 应用(自动选取空闲端口),返回子进程对象。"""
|
||||
# 【环境兼容】WorkBuddy 等 shell 会注入 NODE_OPTIONS(含 --use-system-ca),
|
||||
# Electron 内置 Node 拒绝该 flag 导致安能启动即退出(rc=9)。
|
||||
# 拉起前从子进程环境里摘掉 NODE_OPTIONS。
|
||||
anneng_env = os.environ.copy()
|
||||
anneng_env.pop("NODE_OPTIONS", None)
|
||||
anneng_env = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in _ANNENG_STRIP_EXACT and not key.startswith(_ANNENG_STRIP_PREFIXES)
|
||||
}
|
||||
port = _find_free_port()
|
||||
print(f">> 以调试模式启动【安能】应用(端口 {port}):{app_path}")
|
||||
proc = subprocess.Popen(
|
||||
@@ -150,19 +148,6 @@ def probe_site_login(site_name, pages_map):
|
||||
return False
|
||||
|
||||
|
||||
def probe_data_file(site_name, kind):
|
||||
"""探测单站应到/实到数据文件是否存在且为今天。返回 (is_today, mtime_str)。"""
|
||||
fname = DATA_FILENAMES.get(site_name, {}).get(kind, "")
|
||||
if not fname:
|
||||
return (False, "")
|
||||
path = os.path.join(DOWNLOAD_DIR, fname)
|
||||
if not os.path.exists(path):
|
||||
return (False, "")
|
||||
dt = datetime.fromtimestamp(os.path.getmtime(path))
|
||||
is_today = dt.date() == datetime.now().date()
|
||||
return (is_today, dt.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
|
||||
|
||||
# ============================ 运行上下文 ============================
|
||||
|
||||
|
||||
@@ -179,6 +164,7 @@ class RuntimeContext:
|
||||
sites_to_watch,
|
||||
debug_mode,
|
||||
debug_target,
|
||||
foreground=True,
|
||||
):
|
||||
self.pw = pw
|
||||
self.browser = browser
|
||||
@@ -188,6 +174,8 @@ class RuntimeContext:
|
||||
self.sites_to_watch = sites_to_watch
|
||||
self.debug_mode = debug_mode
|
||||
self.debug_target = debug_target
|
||||
# True=任务执行时把 page 置顶(交互调试);False=后台静默不置顶(服务模式,避免抢焦点)
|
||||
self.foreground = foreground
|
||||
|
||||
def stop(self):
|
||||
"""关闭 browser + 安能 + Playwright。退出时调用。"""
|
||||
@@ -236,11 +224,15 @@ def seed_legacy_config():
|
||||
print(f">> [seed] 从 config.yaml 灌入站点配置: {', '.join(seeded)}")
|
||||
|
||||
|
||||
def launch_and_prepare(debug_mode=False, debug_target=""):
|
||||
def launch_and_prepare(debug_mode=False, debug_target="", foreground=True):
|
||||
"""启动 Playwright + 各站 page + 就绪轮询 + 弹窗清理 + 心跳初值,返回 RuntimeContext。
|
||||
|
||||
必须在"持有 Playwright 的线程"调用(交互模式主线程 / 服务模式 worker 线程)。
|
||||
阻塞至所有站点登录就绪才返回。
|
||||
|
||||
foreground:True=任务执行时把 page 置顶(交互调试);False=后台静默不置顶(服务模式,
|
||||
避免抢用户焦点)。仅控制任务执行阶段的 bring_to_front;启动登录/初始弹窗清理的置顶
|
||||
不受影响(启动时窗口需对用户可见以便登录)。
|
||||
"""
|
||||
# 0. 状态库建表/迁移 + 从 config.yaml 灌入站点配置(须在 reset_login_states 等之前)
|
||||
state_store.init_db()
|
||||
@@ -294,6 +286,15 @@ def launch_and_prepare(debug_mode=False, debug_target=""):
|
||||
_launch_args = ["--remote-debugging-port=9223"] if debug_mode else []
|
||||
browser = pw.chromium.launch(headless=False, args=_launch_args)
|
||||
context = browser.new_context(viewport={"width": 1920, "height": 1080})
|
||||
# 默认禁用麦克风/摄像头:在每个页面/iframe 加载前覆盖 getUserMedia 为“直接拒绝”,
|
||||
# 这样站点(如韵达登录/工作台会请求麦克风)调用时立即 NotAllowedError,Chromium 不再
|
||||
# 弹出系统授权窗,且麦克风被真正挡住(不是授权给它)。物流工作台无需音视频采集。
|
||||
context.add_init_script(
|
||||
"(()=>{const d=()=>Promise.reject(new DOMException('Permission disabled','NotAllowedError'));"
|
||||
"if(navigator.mediaDevices)navigator.mediaDevices.getUserMedia=d;"
|
||||
"for(const k of ['getUserMedia','webkitGetUserMedia','mozGetUserMedia']){"
|
||||
"if(typeof navigator[k]==='function')navigator[k]=function(){return d();};}})();"
|
||||
)
|
||||
|
||||
pages_map = {}
|
||||
print("\n====================================================")
|
||||
@@ -397,6 +398,7 @@ def launch_and_prepare(debug_mode=False, debug_target=""):
|
||||
sites_to_watch,
|
||||
debug_mode,
|
||||
debug_target,
|
||||
foreground,
|
||||
)
|
||||
|
||||
|
||||
@@ -440,38 +442,94 @@ def _dismiss_initial_popups(pages_map):
|
||||
|
||||
|
||||
def _web_handler(site, download_func):
|
||||
"""构造网页站任务 handler:单 page 先 bring_to_front 再 download;顺心(list) 直接传。"""
|
||||
"""构造网页站任务 handler。
|
||||
|
||||
def handler(ctx):
|
||||
foreground(ctx)控制任务执行时是否把 page 置顶:服务模式后台跑不置顶,避免抢用户
|
||||
焦点;交互模式置顶便于调试。顺心是 page 列表,置顶标志透传给 shunxin_download。
|
||||
"""
|
||||
|
||||
def handler(ctx, force=False, date=None):
|
||||
pg = ctx.pages_map[site]
|
||||
if not isinstance(pg, list):
|
||||
if isinstance(pg, list):
|
||||
# 顺心双账号:置顶与否交给 shunxin_download 在逐账号循环里按 foreground 决定
|
||||
return download_func(pg, foreground=ctx.foreground, force=force, date=date)
|
||||
if ctx.foreground:
|
||||
pg.bring_to_front()
|
||||
return download_func(pg)
|
||||
return download_func(pg, force=force, date=date)
|
||||
|
||||
return handler
|
||||
|
||||
|
||||
def _site_undelivered_handler(site):
|
||||
"""4 站未到:下应到+实到 → 比对写 downloads/<站>-未到数据.xlsx。
|
||||
任一下载失败 → 清掉旧未到文件、返回 False(前端不展示陈旧未到)。"""
|
||||
"""4 站未到:下应到+实到 → DB 比对 → 写 output/<站>-<日期>-未到数据.xlsx。
|
||||
应到全量去重(已落库则跳过导出),因此比对不依赖 Excel 文件,走数据库查询。
|
||||
下载成功则返回 True(比对失败不影响任务判定,数据已入库)。"""
|
||||
|
||||
def handler(ctx):
|
||||
# 各站下载入口约定返回 True/False;顺心历史返回 None(视为成功,与 dispatch 一致)
|
||||
exp_ok = TASK_HANDLERS[(site, "expected")](ctx) is not False
|
||||
def handler(ctx, force=False, date=None):
|
||||
exp_ok = TASK_HANDLERS[(site, "expected")](ctx, force, date) is not False
|
||||
act_ok = (
|
||||
(TASK_HANDLERS[(site, "actual")](ctx) is not False) if exp_ok else False
|
||||
(TASK_HANDLERS[(site, "actual")](ctx, force, date) is not False)
|
||||
if exp_ok
|
||||
else False
|
||||
)
|
||||
if exp_ok and act_ok:
|
||||
return compare.write_site_file(site)
|
||||
stale = os.path.join(DOWNLOAD_DIR, SITE_UNDELIVERED_FILE.format(name=site))
|
||||
if os.path.exists(stale):
|
||||
os.remove(stale)
|
||||
return False
|
||||
if not exp_ok or not act_ok:
|
||||
return False
|
||||
|
||||
# ── 先入库再比对(修复时序:比对须读到本次下载的数据,
|
||||
# 否则首次/force 时 PG 无当天数据,比对返回 None、不产出 Excel)──
|
||||
try:
|
||||
_record_business_date(site, "undelivered", date)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from inbound_verify import store # 懒导入,避免成环
|
||||
|
||||
if store.ingest_enabled():
|
||||
store.ingest_task(
|
||||
site, "undelivered"
|
||||
) # 4 站 = ingest expected + actual
|
||||
print(f">> [入库] {site} 前置入库完成")
|
||||
except Exception as e:
|
||||
print(f">> [入库] {site} 前置入库失败(不影响比对尝试): {e}")
|
||||
|
||||
# ── DB 比对(替代旧 Excel 比对)──
|
||||
try:
|
||||
from inbound_verify import db_compare # 懒导入,避免成环
|
||||
|
||||
if date:
|
||||
target_date = date
|
||||
else:
|
||||
offset = state_store.get_offset(site, "actual")
|
||||
target_date = (datetime.now().date() - timedelta(days=offset)).strftime(
|
||||
"%Y-%m-%d"
|
||||
)
|
||||
|
||||
result = db_compare.compare_site_date(site, target_date)
|
||||
if result is not None:
|
||||
db_compare.write_result_excel(result)
|
||||
else:
|
||||
print(f">> [未到] {site} {target_date}: 当天无实到数据,跳过比对")
|
||||
except Exception as e:
|
||||
print(f">> [未到] {site} DB 比对异常(不影响下载结果): {e}")
|
||||
|
||||
return True # 下载成功即返回 True,比对失败不影响任务判定
|
||||
|
||||
return handler
|
||||
|
||||
|
||||
# 「跑比对」= 纯离线比对(用 downloads/ 现有文件生成全站汇总;下载交由各站定时/手动)。
|
||||
# 「跑比对」= DB 版全站汇总报表(替代旧 compare.main Excel 路径;下载交由各站定时/手动)。
|
||||
|
||||
|
||||
def _run_db_full_report(date=None):
|
||||
"""生成 DB 版全站汇总报表(output/应到未到数据.xlsx)。
|
||||
懒导入 db_compare,best-effort:失败只告警,返回 True(与旧 lambda 契约一致)。"""
|
||||
try:
|
||||
from inbound_verify import db_compare
|
||||
|
||||
db_compare.build_full_report(date)
|
||||
except Exception as e:
|
||||
print(f">> [跑比对] DB 汇总报表生成失败: {e}")
|
||||
return True
|
||||
|
||||
|
||||
TASK_HANDLERS = {
|
||||
@@ -487,47 +545,173 @@ TASK_HANDLERS = {
|
||||
("韵达", "expected"): _web_handler("韵达", yunda.yunda_expected_download),
|
||||
("韵达", "actual"): _web_handler("韵达", yunda.yunda_actual_download),
|
||||
("韵达", "undelivered"): _site_undelivered_handler("韵达"),
|
||||
("安能", "expected"): lambda ctx: anneng.anneng_expected_download(),
|
||||
("安能", "actual"): lambda ctx: anneng.anneng_actual_download(),
|
||||
(
|
||||
"安能",
|
||||
"expected",
|
||||
): lambda ctx, force=False, date=None: anneng.anneng_expected_download(
|
||||
force=force, date=date
|
||||
),
|
||||
(
|
||||
"安能",
|
||||
"actual",
|
||||
): lambda ctx, force=False, date=None: anneng.anneng_actual_download(
|
||||
force=force, date=date
|
||||
),
|
||||
("安能", "undelivered"): _site_undelivered_handler("安能"),
|
||||
("__compare__", "compare"): lambda ctx: (compare.main() or True),
|
||||
("__compare__", "compare"): lambda ctx, force=False, date=None: _run_db_full_report(
|
||||
date
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _record_business_date(site, kind):
|
||||
def _record_business_date(site, kind, date=None):
|
||||
"""下载成功后,把本次数据的业务日期快照写进状态库(供前端/报告显示「是哪天的数据」)。
|
||||
业务日期 = 下载当天 − 该数据对应的日期偏移。__compare__ 无数据概念,跳过。
|
||||
有 date 用 date;否则 = 下载当天 − 该数据对应的日期偏移。__compare__ 无数据概念,跳过。
|
||||
|
||||
kind → 写入:
|
||||
expected/actual:各写自己一列(偏移各取其列)。
|
||||
undelivered:百世直供(恒 0)写 undelivered;4 站未到由 _site_undelivered_handler
|
||||
expected/actual:各写自己一列。
|
||||
undelivered:百世直供(恒当天)写 undelivered;4 站未到由 _site_undelivered_handler
|
||||
内部连带下了 expected+actual(不经 dispatch,无业务日期写入),故此处一并补写
|
||||
expected/actual/undelivered 三列——actual 用 actual 偏移、未到跟随 expected 偏移。
|
||||
顺带置 ready=True,让前端不必等心跳即可反映下载成功;写入失败仅告警、不影响任务判定。"""
|
||||
只写业务日期;ready 语义已移交「入库成功」(_persist_to_db 置位),此处不再碰 ready。"""
|
||||
if site == "__compare__":
|
||||
return
|
||||
today = datetime.now().date()
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def _write(k, off):
|
||||
biz = (today - timedelta(days=off)).strftime("%Y-%m-%d")
|
||||
def _write(k, biz_or_off):
|
||||
# biz_or_off: int=偏移(today−off);str=已确定业务日期(date)
|
||||
biz = (
|
||||
(today - timedelta(days=biz_or_off)).strftime("%Y-%m-%d")
|
||||
if isinstance(biz_or_off, int)
|
||||
else biz_or_off
|
||||
)
|
||||
try:
|
||||
state_store.set_data_state(
|
||||
site, k, ready=True, generated_at=now, business_date=biz
|
||||
)
|
||||
state_store.set_business_date(site, k, biz)
|
||||
except Exception as e:
|
||||
print(f">> [状态] 写业务日期失败 {site}/{k}: {e}")
|
||||
|
||||
def off(kind_key):
|
||||
return state_store.get_offset(site, kind_key)
|
||||
|
||||
if kind == "expected":
|
||||
_write("expected", state_store.get_offset(site, "expected"))
|
||||
_write("expected", date if date else off("expected"))
|
||||
elif kind == "actual":
|
||||
_write("actual", state_store.get_offset(site, "actual"))
|
||||
_write("actual", date if date else off("actual"))
|
||||
elif site == "百世":
|
||||
_write("undelivered", 0)
|
||||
else: # 4 站 undelivered:连带补写 expected/actual/undelivered 三列
|
||||
_write("expected", state_store.get_offset(site, "expected"))
|
||||
_write("actual", state_store.get_offset(site, "actual"))
|
||||
_write("undelivered", state_store.get_offset(site, "expected"))
|
||||
_write("expected", date if date else off("expected"))
|
||||
_write("actual", date if date else off("actual"))
|
||||
_write("undelivered", date if date else off("expected"))
|
||||
|
||||
|
||||
def _ready_flags(site):
|
||||
"""从 PG 业务表派生单站三就绪态(ready = DB 数据真相)。
|
||||
|
||||
expected/actual = PG 中存在对应 target_date(today − offset)的数据;
|
||||
百世 undelivered = baishi_daily_stats 中存在 target_date 的数据;
|
||||
4 站 undelivered = expected_ready ∧ actual_ready(派生)。
|
||||
PG 不可达时返回全 False(降级安全,不阻塞心跳)。
|
||||
|
||||
返回 (flags: {kind: bool}, dates: {kind: target_date_str})。
|
||||
dates 与 flags 同源——ready=True 时 business_date 即该 target_date,
|
||||
彻底消除 ready 与 business_date 不同源导致的日期标签漂移。"""
|
||||
from inbound_verify import store # 懒导入:避免模块级循环
|
||||
|
||||
today = date.today()
|
||||
today_str = today.isoformat()
|
||||
|
||||
if site == "百世":
|
||||
has_und, _ = store.has_data(site, "undelivered", today_str)
|
||||
return (
|
||||
{"expected": False, "actual": False, "undelivered": has_und},
|
||||
{"undelivered": today_str},
|
||||
)
|
||||
|
||||
exp_off = state_store.get_offset(site, "expected")
|
||||
act_off = state_store.get_offset(site, "actual")
|
||||
exp_date = (today - timedelta(days=exp_off)).isoformat()
|
||||
act_date = (today - timedelta(days=act_off)).isoformat()
|
||||
|
||||
has_exp, _ = store.has_data(site, "expected", exp_date)
|
||||
has_act, _ = store.has_data(site, "actual", act_date)
|
||||
return (
|
||||
{"expected": has_exp, "actual": has_act, "undelivered": has_exp and has_act},
|
||||
{"expected": exp_date, "actual": act_date, "undelivered": exp_date},
|
||||
)
|
||||
|
||||
|
||||
def _apply_ready(site, flags, dates=None):
|
||||
"""写入单站就绪态 + 业务日期(同源:ready 与 business_date 均据 PG + offset 派生)。
|
||||
ready=True 时同步写入 target_date 作为 business_date,消除不同源导致的日期标签漂移。
|
||||
失败仅告警。"""
|
||||
for k, rdy in flags.items():
|
||||
try:
|
||||
state_store.set_ready(site, k, rdy)
|
||||
if rdy and dates and dates.get(k):
|
||||
state_store.set_business_date(site, k, dates[k])
|
||||
except Exception as e:
|
||||
print(f">> [状态] 置就绪态失败 {site}/{k}: {e}")
|
||||
|
||||
|
||||
def capture_error_screenshot(page, site, kind, attempt, error):
|
||||
"""流程失败时截取当前页面,保存到 logs/screenshots/。
|
||||
page: Playwright Page 对象(安能传 None 走 CDP 分支,调用方自行处理)。
|
||||
截图失败绝不外抛——只打告警,不干扰任务重试/清场流程。"""
|
||||
try:
|
||||
os.makedirs(SCREENSHOT_DIR, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
err_short = (error or "unknown")[:40].replace("/", "_").replace("\\", "_")
|
||||
fname = f"{site}_{kind}_{ts}_attempt{attempt}_{err_short}.png"
|
||||
path = os.path.join(SCREENSHOT_DIR, fname)
|
||||
page.screenshot(path=path, full_page=False)
|
||||
print(f"📸 【{site}-{kind}】错误截图已保存: {path}")
|
||||
except Exception as se:
|
||||
print(f"📸 【{site}-{kind}】截图失败(不影响任务): {se}")
|
||||
|
||||
|
||||
def _refresh_ready(site):
|
||||
"""入库后立即据 PG 派生并写入该站就绪态(省 30s 心跳等待,与心跳同源)。"""
|
||||
flags, dates = _ready_flags(site)
|
||||
_apply_ready(site, flags, dates)
|
||||
|
||||
|
||||
def _persist_to_db(site, kind):
|
||||
"""下载成功后把本次数据入库 PostgreSQL(尽力而为,绝不外抛,不影响任务判定)。
|
||||
- __compare__ 无源数据,跳过。
|
||||
- auto_ingest=false 时跳过(无 PG/cpolar 的开发机)。
|
||||
- 懒导入 store 以回避 import 顺序(store↔compare 与 runtime↔compare 共存)。
|
||||
- 结果写 state_store.ingest_state,供 /api/status 反映入库健康。
|
||||
所有写库/写状态都包 try/except:失败仅告警,绝不改变 dispatch_task 的 SUCCESS 判定。"""
|
||||
if site == "__compare__":
|
||||
return
|
||||
try:
|
||||
from inbound_verify import store # 懒导入:冷路径(每下载一次),回避成环
|
||||
except Exception as e:
|
||||
print(f">> [warn] 入库模块不可用: {e}")
|
||||
return
|
||||
try:
|
||||
if not store.ingest_enabled(): # 移入 try:config.yaml 缺失/损坏时也不外抛
|
||||
print(">> [入库] 已关闭 (auto_ingest=false),跳过")
|
||||
return
|
||||
count = store.ingest_task(site, kind)
|
||||
# 4 站 undelivered 连带入了 expected+actual:按实际入库的类补记 ingest_state,
|
||||
# 否则心跳派生 ready(expected ∧ actual → undelivered)会读到陈旧值。
|
||||
logged = (
|
||||
["expected", "actual", "undelivered"]
|
||||
if kind == "undelivered" and site != "百世"
|
||||
else [kind]
|
||||
)
|
||||
for k in logged:
|
||||
state_store.set_ingest_state(site, k, ok=True, count=count)
|
||||
_refresh_ready(site) # 入库成功 → 立即据 ingest_state 派生就绪态(与心跳同源)
|
||||
print(f">> [入库] {site}/{kind} 成功,{count} 条")
|
||||
except Exception as e:
|
||||
print(f">> [warn] 入库失败 {site}/{kind}: {e}")
|
||||
try:
|
||||
state_store.set_ingest_state(site, kind, ok=False, error=str(e))
|
||||
except Exception as e2:
|
||||
print(f">> [warn] 写入库状态也失败: {e2}")
|
||||
|
||||
|
||||
def dispatch_task(ctx, task_spec):
|
||||
@@ -550,10 +734,11 @@ def dispatch_task(ctx, task_spec):
|
||||
if handler is None:
|
||||
return (state_store.TASK_FAILED, f"未知任务: {site}/{kind}")
|
||||
try:
|
||||
ret = handler(ctx)
|
||||
ret = handler(ctx, bool(task_spec.get("force", False)), task_spec.get("date"))
|
||||
if ret is False:
|
||||
return (state_store.TASK_FAILED, "任务执行失败(重试耗尽)")
|
||||
_record_business_date(site, kind)
|
||||
_record_business_date(site, kind, task_spec.get("date"))
|
||||
_persist_to_db(site, kind)
|
||||
return (state_store.TASK_SUCCESS, None)
|
||||
except Exception as e:
|
||||
return (state_store.TASK_FAILED, str(e))
|
||||
@@ -563,8 +748,11 @@ def dispatch_task(ctx, task_spec):
|
||||
|
||||
|
||||
def run_heartbeat(ctx):
|
||||
"""一轮心跳:探测各站登录态 + 数据文件,写状态库;登录态变化时提示。
|
||||
"""一轮心跳:探测各站登录态 + 据 PG 业务表派生数据就绪态;登录态变化时提示。
|
||||
|
||||
ready 直接查询 PG 业务表(expected_record / actual_record / baishi_daily_stats),
|
||||
以「目标业务日期是否有数据」为唯一依据,彻底消除 ingest_state 日期比对带来的每日零点重置。
|
||||
_refresh_ready 在入库瞬间即据 PG 派生(省 30s 等待),心跳同源复核。
|
||||
只在 Playwright 所属线程调用。
|
||||
"""
|
||||
prev = state_store.get_all_status()
|
||||
@@ -575,6 +763,5 @@ def run_heartbeat(ctx):
|
||||
now_login = state_store.LOGIN_IN if logged_in else state_store.LOGIN_OUT
|
||||
if prev_login and prev_login not in (now_login, state_store.LOGIN_UNKNOWN):
|
||||
print(f"\n ⚠️【{site_name}】登录态变化: {prev_login} → {now_login}")
|
||||
for kind in ("expected", "actual", "undelivered"):
|
||||
ready, gen_at = probe_data_file(site_name, kind)
|
||||
state_store.set_data_state(site_name, kind, ready, gen_at)
|
||||
flags, dates = _ready_flags(site_name)
|
||||
_apply_ready(site_name, flags, dates)
|
||||
|
||||
@@ -43,10 +43,37 @@ from inbound_verify.paths import DOWNLOAD_DIR, CONFIG_PATH
|
||||
from inbound_verify import state_store
|
||||
|
||||
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
def _capture_error_screenshot(site, kind, attempt, error):
|
||||
"""安能 CDP 错误截图(best-effort;失败仅告警,绝不外抛)。"""
|
||||
try:
|
||||
import base64, os
|
||||
from datetime import datetime
|
||||
from inbound_verify.paths import SCREENSHOT_DIR
|
||||
|
||||
os.makedirs(SCREENSHOT_DIR, exist_ok=True)
|
||||
pages = list_pages()
|
||||
if not pages:
|
||||
return
|
||||
cdp = CDP(pages[0]["webSocketDebuggerUrl"])
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
err_short = (error or "unknown")[:40].replace("/", "_").replace("\\", "_")
|
||||
fname = f"{site}_{kind}_{ts}_attempt{attempt}_{err_short}.png"
|
||||
path = os.path.join(SCREENSHOT_DIR, fname)
|
||||
result = cdp.call("Page.captureScreenshot", format="png")
|
||||
with open(path, "wb") as f:
|
||||
f.write(base64.b64decode(result["data"]))
|
||||
cdp.close()
|
||||
print(f"📸 【{site}-{kind}】错误截图已保存: {path}")
|
||||
except Exception as se:
|
||||
print(f"📸 【{site}-{kind}】截图失败(不影响任务): {se}")
|
||||
|
||||
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3, page=None):
|
||||
"""异常兜底:flow 失败 → 重置回初始态 → 重试,最多 max_attempts 次(含首次)。
|
||||
|
||||
每次失败都重置(含最终放弃那一次):既是重试前的清场,也保证最终放弃时环境干净。
|
||||
最后一次失败时(重置前)自动截图保存到 logs/screenshots/,供问题排查。
|
||||
(安能通过 CDP 截图,page 参数忽略;保留为统一签名兼容。)
|
||||
flow 为零参可调用;返回 False 视为失败,其余视为成功。
|
||||
返回 True=最终成功,False=重试耗尽放弃(供调度层判断任务成败)。
|
||||
"""
|
||||
@@ -60,6 +87,11 @@ def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"⚠️ 【{site_name}-{label}】第 {attempt}/{max_attempts} 次失败: {e}")
|
||||
if attempt == max_attempts:
|
||||
try:
|
||||
_capture_error_screenshot(site_name, label, attempt, str(e))
|
||||
except Exception:
|
||||
pass
|
||||
print(f" → 重置【{site_name}】到初始态,清理环境 ...")
|
||||
try:
|
||||
reset()
|
||||
@@ -112,7 +144,10 @@ class CDP:
|
||||
"""绑定到单个页面目标的同步 CDP 客户端。"""
|
||||
|
||||
def __init__(self, ws_url):
|
||||
self.ws = websocket.create_connection(ws_url)
|
||||
# socket 级超时:Electron 业务 tab 偶发不回包时,recv 最多卡 15s 即抛
|
||||
# WebSocketTimeoutException,让上层 wait_until/with_retry 能失败→重试,
|
||||
# 而不是无限阻塞(曾导致安能下载卡死 ~22 分钟、轮询 300s 截止也无法触发)。
|
||||
self.ws = websocket.create_connection(ws_url, timeout=15)
|
||||
self._id = 0
|
||||
self.call("Runtime.enable")
|
||||
|
||||
@@ -852,13 +887,18 @@ def _load_query_days():
|
||||
return max(1, days)
|
||||
|
||||
|
||||
def anneng_expected_download():
|
||||
def anneng_expected_download(force=False, date=None):
|
||||
"""安能:应到货物数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry("安能", "应到", anneng_expected_download_impl, anneng_reset)
|
||||
return with_retry(
|
||||
"安能",
|
||||
"应到",
|
||||
lambda: anneng_expected_download_impl(force=force, date=date),
|
||||
anneng_reset,
|
||||
)
|
||||
|
||||
|
||||
def anneng_expected_download_impl():
|
||||
def anneng_expected_download_impl(force=False, date=None):
|
||||
"""安能:应到货物数据(运单信息)下载,完整流程(单次执行,无重试;供自动化测试用)。"""
|
||||
print("\n▶ 开始执行【安能 - 应到货物数据下载】任务 ...")
|
||||
download_dir = DOWNLOAD_DIR
|
||||
@@ -869,14 +909,32 @@ def anneng_expected_download_impl():
|
||||
|
||||
offset = state_store.get_offset("安能")
|
||||
today = datetime.now()
|
||||
target = today - timedelta(days=offset)
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_str = f"{target.year}-{target.month:02d}-{target.day:02d}"
|
||||
start_str = target_str
|
||||
today_str = target_str
|
||||
print(f">> 查询日期: [{target_str}](偏移 {offset},0=今天)")
|
||||
src = f"指定 {date}" if date else f"偏移 {offset},0=今天"
|
||||
print(f">> 查询日期: [{target_str}]({src})")
|
||||
|
||||
main_cdp = find_main_page_cdp()
|
||||
export_times = []
|
||||
|
||||
# 【去重】加载本站已落库交接单号;force=True 或查询失败时 existing=空集(不去重)
|
||||
if force:
|
||||
existing = set()
|
||||
print(">> [去重] 强制重下,跳过去重。")
|
||||
else:
|
||||
try:
|
||||
from inbound_verify import store
|
||||
|
||||
existing = store.get_existing_handover_nos("安能")
|
||||
except Exception as _e:
|
||||
existing = set()
|
||||
print(f">> [去重] 加载已落库交接单号失败,本次不去重: {_e}")
|
||||
|
||||
try:
|
||||
# 1) 确认主页就绪并导航到“进站交接单查询”
|
||||
wait_home_ready(main_cdp)
|
||||
@@ -915,6 +973,10 @@ def anneng_expected_download_impl():
|
||||
|
||||
for i, ewbs_no in enumerate(target_ids, start=1):
|
||||
print(f" ⏳ [{i}/{len(target_ids)}] 交接单号 {ewbs_no}")
|
||||
# 【去重】已落库则跳过:不双击、不导出、不 append export_times
|
||||
if ewbs_no in existing:
|
||||
print(f" ⏭️ 交接单号 {ewbs_no} 已落库,跳过。")
|
||||
continue
|
||||
activate_tab(tab_cdp, "交接单信息")
|
||||
time.sleep(0.3)
|
||||
if not dblclick_jiaojie_dan_row(tab_cdp, ewbs_no):
|
||||
@@ -933,6 +995,11 @@ def anneng_expected_download_impl():
|
||||
close_tab_by_label(main_cdp, "进站交接单查询")
|
||||
time.sleep(0.8)
|
||||
|
||||
# 【去重兜底】全部已落库/无数据 → 无导出任务,查询 tab 已关,跳过下载段
|
||||
if not export_times:
|
||||
print(">> 本次无新交接单需导出(全部已落库或无数据),结束。")
|
||||
return True
|
||||
|
||||
# 5) 打开导出下载 tab,轮询并下载
|
||||
print(">> 打开【导出下载】tab ...")
|
||||
export_cdp = ensure_tab_open(main_cdp, "导出下载", EXPORT_TAB_URL_HINT)
|
||||
@@ -1230,13 +1297,15 @@ def _save_actual(rows, download_dir):
|
||||
print("====================================================")
|
||||
|
||||
|
||||
def anneng_actual_download():
|
||||
def anneng_actual_download(force=False, date=None):
|
||||
"""安能:实到数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry("安能", "实到", anneng_actual_download_impl, anneng_reset)
|
||||
return with_retry(
|
||||
"安能", "实到", lambda: anneng_actual_download_impl(date=date), anneng_reset
|
||||
)
|
||||
|
||||
|
||||
def anneng_actual_download_impl():
|
||||
def anneng_actual_download_impl(date=None):
|
||||
"""安能:实到数据(网点到件扫描,子单)下载,完整流程(单次执行,无重试;供自动化测试用)。"""
|
||||
print("\n▶ 开始执行【安能 - 实到货物数据下载】任务 ...")
|
||||
download_dir = DOWNLOAD_DIR
|
||||
@@ -1247,10 +1316,14 @@ def anneng_actual_download_impl():
|
||||
|
||||
offset = state_store.get_offset("安能", "actual")
|
||||
today = datetime.now()
|
||||
target = today - timedelta(days=offset)
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
start_str = f"{target.year}/{target.month:02d}/{target.day:02d} 00:00:00"
|
||||
end_str = f"{target.year}/{target.month:02d}/{target.day:02d} 23:59:59"
|
||||
print(f">> 扫描日期: [{start_str} 至 {end_str}](偏移 {offset},0=今天)")
|
||||
src = f"指定 {date}" if date else f"偏移 {offset},0=今天"
|
||||
print(f">> 扫描日期: [{start_str} 至 {end_str}]({src})")
|
||||
|
||||
main_cdp = find_main_page_cdp()
|
||||
try:
|
||||
|
||||
@@ -8,10 +8,11 @@ from inbound_verify.paths import DOWNLOAD_DIR, CONFIG_PATH
|
||||
from inbound_verify import state_store
|
||||
|
||||
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3, page=None):
|
||||
"""异常兜底:flow 失败 → 重置回初始态 → 重试,最多 max_attempts 次(含首次)。
|
||||
|
||||
每次失败都重置(含最终放弃那一次):既是重试前的清场,也保证最终放弃时环境干净。
|
||||
最后一次失败时(重置前)自动截图保存到 logs/screenshots/,供问题排查。
|
||||
flow 为零参可调用;返回 False 视为失败,其余视为成功。
|
||||
返回 True=最终成功,False=重试耗尽放弃(供调度层判断任务成败)。
|
||||
"""
|
||||
@@ -25,6 +26,13 @@ def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"⚠️ 【{site_name}-{label}】第 {attempt}/{max_attempts} 次失败: {e}")
|
||||
if attempt == max_attempts and page is not None:
|
||||
try:
|
||||
from inbound_verify.runtime import capture_error_screenshot
|
||||
|
||||
capture_error_screenshot(page, site_name, label, attempt, str(e))
|
||||
except Exception:
|
||||
pass
|
||||
print(f" → 重置【{site_name}】到初始态,清理环境 ...")
|
||||
try:
|
||||
reset()
|
||||
@@ -137,14 +145,17 @@ def _close_tab(page, tab_name):
|
||||
print(f" ⚠️ 关闭标签页【{tab_name}】时出错: {e}")
|
||||
|
||||
|
||||
def baishi_download_undelivered_data(page):
|
||||
"""百世:一键提取应到未到(当日未扫)数据(内部含异常兜底重试,路由层无感)。"""
|
||||
def baishi_download_undelivered_data(page, force=False, date=None):
|
||||
"""百世:一键提取应到未到(当日未扫)数据(内部含异常兜底重试,路由层无感)。
|
||||
|
||||
date 形参仅为对齐统一透传签名(百世固定下载当天),忽略。"""
|
||||
|
||||
return with_retry(
|
||||
"百世",
|
||||
"应到未到",
|
||||
lambda: baishi_download_undelivered_data_impl(page),
|
||||
lambda: baishi_reset(page),
|
||||
page=page,
|
||||
)
|
||||
|
||||
|
||||
@@ -198,6 +209,11 @@ def baishi_download_undelivered_data_impl(page):
|
||||
if _exp_n > 0:
|
||||
state_store.set_setting("百世", "scan_expected_pieces", str(_exp_n))
|
||||
state_store.set_setting("百世", "scan_arrived_pieces", str(_arr_n))
|
||||
from inbound_verify import (
|
||||
store,
|
||||
) # 直接落库 PG(一步,不绕 state_store→store)
|
||||
|
||||
store.upsert_baishi_daily_stats(_exp_n, _arr_n)
|
||||
print(f" ℹ️ 已记录百世应到/实到基数:应扫 {_exp_n} / 已扫 {_arr_n}")
|
||||
except Exception as _e:
|
||||
# 抓取失败绝不影响未到明细下载主流程
|
||||
|
||||
@@ -11,10 +11,11 @@ from inbound_verify.paths import DOWNLOAD_DIR, CONFIG_PATH
|
||||
from inbound_verify import state_store
|
||||
|
||||
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3, page=None):
|
||||
"""异常兜底:flow 失败 → 重置回初始态 → 重试,最多 max_attempts 次(含首次)。
|
||||
|
||||
每次失败都重置(含最终放弃那一次):既是重试前的清场,也保证最终放弃时环境干净。
|
||||
最后一次失败时(重置前)自动截图保存到 logs/screenshots/,供问题排查。
|
||||
flow 为零参可调用;返回 False 视为失败,其余视为成功。
|
||||
返回 True=最终成功,False=重试耗尽放弃(供调度层判断任务成败)。
|
||||
"""
|
||||
@@ -28,6 +29,13 @@ def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"⚠️ 【{site_name}-{label}】第 {attempt}/{max_attempts} 次失败: {e}")
|
||||
if attempt == max_attempts and page is not None:
|
||||
try:
|
||||
from inbound_verify.runtime import capture_error_screenshot
|
||||
|
||||
capture_error_screenshot(page, site_name, label, attempt, str(e))
|
||||
except Exception:
|
||||
pass
|
||||
print(f" → 重置【{site_name}】到初始态,清理环境 ...")
|
||||
try:
|
||||
reset()
|
||||
@@ -88,6 +96,64 @@ def _remove_if_exists(path):
|
||||
pass
|
||||
|
||||
|
||||
def _shunxin_navigate_picker_to_target(page, date_str, max_flips=24):
|
||||
"""顺心 Ant Design 单月面板跨月导航(车辆点到 / 卸车扫描记录 共用同一组件)。
|
||||
|
||||
读面板头部 .ant-picker-year-btn / .ant-picker-month-btn 得当前显示的年月,按与目标
|
||||
年月的差值点 .ant-picker-header-prev-btn(上月)/ .ant-picker-header-next-btn(下月),
|
||||
翻到目标月视窗。返回 True 表示当前视窗已是目标月(目标格子随后可见可点)。
|
||||
调用前提:开始/结束时间输入已点开,.ant-picker-dropdown:visible 已就绪。
|
||||
"""
|
||||
try:
|
||||
ty, tm = (int(x) for x in date_str.split("-")[:2])
|
||||
except Exception:
|
||||
return True # 解析不出就不翻,交由后续 cell.click 自行成败
|
||||
drop = page.locator(".ant-picker-dropdown:visible")
|
||||
for _ in range(max_flips):
|
||||
try:
|
||||
cur_y = int(
|
||||
re.search(
|
||||
r"\d+", drop.locator(".ant-picker-year-btn").first.inner_text()
|
||||
).group()
|
||||
)
|
||||
cur_m = int(
|
||||
re.search(
|
||||
r"\d+", drop.locator(".ant-picker-month-btn").first.inner_text()
|
||||
).group()
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
cur = cur_y * 12 + (cur_m - 1)
|
||||
tgt = ty * 12 + (tm - 1)
|
||||
if cur == tgt:
|
||||
return True
|
||||
btn_sel = (
|
||||
".ant-picker-header-prev-btn"
|
||||
if tgt < cur
|
||||
else ".ant-picker-header-next-btn"
|
||||
)
|
||||
drop.locator(btn_sel).first.click()
|
||||
page.wait_for_timeout(300)
|
||||
return False
|
||||
|
||||
|
||||
def _shunxin_pick_date(page, date_str):
|
||||
"""在已打开的顺心 Ant Design 日期浮层上选中指定日期格子(含跨月翻月)。
|
||||
|
||||
目标格子不在当前月视窗(跨月)时,先调 _shunxin_navigate_picker_to_target 翻到目标月,
|
||||
再点格子;同月则直接点。与中通 _zto_flip_to_target_month 思路对称,适配 Ant Design 面板。
|
||||
"""
|
||||
cell = page.locator(f".ant-picker-dropdown:visible td[title='{date_str}']").first
|
||||
if not cell.is_visible():
|
||||
print(f" ℹ️ 目标日期 {date_str} 不在当前月视窗,正在翻月导航 ...")
|
||||
if not _shunxin_navigate_picker_to_target(page, date_str):
|
||||
raise RuntimeError(f"翻月后仍无法定位目标日期格子 {date_str}")
|
||||
cell = page.locator(
|
||||
f".ant-picker-dropdown:visible td[title='{date_str}']"
|
||||
).first
|
||||
cell.click()
|
||||
|
||||
|
||||
def shunxin_belonging(page):
|
||||
"""读取顺心当前账号的归属网点名(仅在首页可见,须在导航离开首页前调用)。
|
||||
|
||||
@@ -149,7 +215,7 @@ def shunxin_merge_final(kind, tags):
|
||||
pass
|
||||
|
||||
|
||||
def shunxin_expected_download(pages):
|
||||
def shunxin_expected_download(pages, foreground=True, force=False, date=None):
|
||||
"""顺心:应到货物数据下载(双账号/双归属地,内部含异常兜底重试与数据融合)。
|
||||
|
||||
pages 为该站点的 page 列表(双账号在同一窗口的各一个标签页)。
|
||||
@@ -169,13 +235,17 @@ def shunxin_expected_download(pages):
|
||||
)
|
||||
|
||||
for idx, (pg, tag) in enumerate(zip(pages, tags), start=1):
|
||||
pg.bring_to_front()
|
||||
if foreground:
|
||||
pg.bring_to_front()
|
||||
print(f"\n========== 顺心 · 账号{idx}({tag})应到数据下载 ==========")
|
||||
ok = with_retry(
|
||||
f"顺心-{tag}",
|
||||
"应到",
|
||||
lambda p=pg, t=tag: shunxin_expected_download_impl(p, out_tag=t),
|
||||
lambda p=pg, t=tag, f=force, d=date: shunxin_expected_download_impl(
|
||||
p, out_tag=t, force=f, date=d
|
||||
),
|
||||
lambda p=pg: shunxin_reset(p),
|
||||
page=pg,
|
||||
)
|
||||
if not ok:
|
||||
return False # 某账号重试耗尽 → 整体失败,不融合(避免部分数据)
|
||||
@@ -184,7 +254,7 @@ def shunxin_expected_download(pages):
|
||||
return True
|
||||
|
||||
|
||||
def shunxin_expected_download_impl(page, out_tag=""):
|
||||
def shunxin_expected_download_impl(page, out_tag="", force=False, date=None):
|
||||
"""顺心:应到货物数据下载(单次执行,无重试;供自动化测试探测原始结果用)。
|
||||
|
||||
out_tag 为归属地标签时,合并产物命名为「顺心-{out_tag}-应到货物数据.xlsx」,
|
||||
@@ -221,27 +291,27 @@ def shunxin_expected_download_impl(page, out_tag=""):
|
||||
# 2. 读取服务端日期偏移(0=今天,1=昨天…),单日范围:起止同日
|
||||
offset = state_store.get_offset("顺心")
|
||||
today = datetime.now()
|
||||
target = today - timedelta(days=offset)
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_str = target.strftime("%Y-%m-%d")
|
||||
start_date_str = target_str
|
||||
today_str = target_str
|
||||
print(f">> 正在设置查询日期: [{target_str}](偏移 {offset},0=今天)...")
|
||||
src = f"指定 {date}" if date else f"偏移 {offset},0=今天"
|
||||
print(f">> 正在设置查询日期: [{target_str}]({src})...")
|
||||
|
||||
# 分两步精准呼出和点击时间控件
|
||||
print(" >> 设置起始时间...")
|
||||
page.get_by_placeholder("开始时间").click()
|
||||
page.wait_for_timeout(500)
|
||||
page.locator(
|
||||
f".ant-picker-dropdown:visible td[title='{start_date_str}']"
|
||||
).first.click()
|
||||
_shunxin_pick_date(page, start_date_str)
|
||||
page.wait_for_timeout(300)
|
||||
|
||||
print(" >> 设置截止时间...")
|
||||
page.get_by_placeholder("结束时间").click()
|
||||
page.wait_for_timeout(500)
|
||||
page.locator(
|
||||
f".ant-picker-dropdown:visible td[title='{today_str}']"
|
||||
).first.click()
|
||||
_shunxin_pick_date(page, today_str)
|
||||
page.wait_for_timeout(300)
|
||||
|
||||
# 确认日期
|
||||
@@ -308,12 +378,44 @@ def shunxin_expected_download_impl(page, out_tag=""):
|
||||
count = waybill_btns.count()
|
||||
print(f">> 共发现 {count} 个班次需要导出。")
|
||||
|
||||
# 【去重】加载本站已落库交接单号;force=True 或查询失败时 existing=空集(不去重)。
|
||||
# 两账号共享同一集合(班次号/交接单号跨归属地不重叠)。
|
||||
if force:
|
||||
existing = set()
|
||||
print(">> [去重] 强制重下,跳过去重。")
|
||||
else:
|
||||
try:
|
||||
from inbound_verify import store
|
||||
|
||||
existing = store.get_existing_handover_nos("顺心")
|
||||
except Exception as _e:
|
||||
existing = set()
|
||||
print(f">> [去重] 加载已落库交接单号失败,本次不去重: {_e}")
|
||||
|
||||
for i in range(count):
|
||||
print(f" ⏳ 正在处理第 {i+1}/{count} 个班次...")
|
||||
|
||||
waybill_btns.nth(i).click()
|
||||
page.locator("label[title='运单查询']").wait_for(state="visible")
|
||||
|
||||
# 【方式1】运单列表界面已加载,读交接单号(RTS 开头)→ 已落库则退回列表跳过。
|
||||
# 交接单号格式 RTS\d{3}WJ\d+(如 RTS023WJ374837),用 [A-Z0-9]+ 连续匹配整段。
|
||||
# 读不到(DOM 变动/未渲染)则 handover_no 为空 → 不跳过(安全降级,继续导出)。
|
||||
handover_no = ""
|
||||
try:
|
||||
_txt = page.locator("text=/RTS\\d+/").first.inner_text(timeout=3000)
|
||||
_m = re.search(r"RTS[A-Z0-9]+", _txt)
|
||||
if _m:
|
||||
handover_no = _m.group(0)
|
||||
except Exception:
|
||||
pass
|
||||
print(f" -> 运单列表交接单号:{handover_no or '(未读到,不去重)'}")
|
||||
if handover_no and handover_no in existing:
|
||||
print(f" ⏭️ 交接单号 {handover_no} 已落库,跳过提交导出。")
|
||||
page.get_by_role("tab", name="车辆点到").click()
|
||||
page.wait_for_timeout(500)
|
||||
continue
|
||||
|
||||
# 4. 执行导出流程
|
||||
page.get_by_role("button", name="export 导出").click()
|
||||
|
||||
@@ -341,6 +443,11 @@ def shunxin_expected_download_impl(page, out_tag=""):
|
||||
_close_tab(page, "运单列表")
|
||||
_close_tab(page, "车辆点到")
|
||||
|
||||
# 【去重兜底】全部已落库/无数据 → 无导出任务,标签页已关,跳过下载轮询
|
||||
if not export_times:
|
||||
print(">> 本次无新班次需导出(全部已落库或无数据),结束。")
|
||||
return True
|
||||
|
||||
# 6. 前往数据导出页面去下载
|
||||
print(">> 正在前往【数据导出】界面...")
|
||||
page.locator("a[href='/dataExport']").click()
|
||||
@@ -476,7 +583,7 @@ def shunxin_expected_download_impl(page, out_tag=""):
|
||||
return False
|
||||
|
||||
|
||||
def shunxin_actual_download(pages):
|
||||
def shunxin_actual_download(pages, foreground=True, force=False, date=None):
|
||||
"""顺心:实到货物数据下载(双账号/双归属地,内部含异常兜底重试与数据融合)。
|
||||
|
||||
与 shunxin_expected_download 同构:读归属地 → 去重校验 → 顺序各账号下载 →
|
||||
@@ -494,13 +601,17 @@ def shunxin_actual_download(pages):
|
||||
)
|
||||
|
||||
for idx, (pg, tag) in enumerate(zip(pages, tags), start=1):
|
||||
pg.bring_to_front()
|
||||
if foreground:
|
||||
pg.bring_to_front()
|
||||
print(f"\n========== 顺心 · 账号{idx}({tag})实到数据下载 ==========")
|
||||
ok = with_retry(
|
||||
f"顺心-{tag}",
|
||||
"实到",
|
||||
lambda p=pg, t=tag: shunxin_actual_download_impl(p, out_tag=t),
|
||||
lambda p=pg, t=tag, d=date: shunxin_actual_download_impl(
|
||||
p, out_tag=t, date=d
|
||||
),
|
||||
lambda p=pg: shunxin_reset(p),
|
||||
page=pg,
|
||||
)
|
||||
if not ok:
|
||||
return False
|
||||
@@ -509,7 +620,7 @@ def shunxin_actual_download(pages):
|
||||
return True
|
||||
|
||||
|
||||
def shunxin_actual_download_impl(page, out_tag=""):
|
||||
def shunxin_actual_download_impl(page, out_tag="", date=None):
|
||||
"""顺心:实到货物数据下载(单次执行,无重试;供自动化测试探测原始结果用)。
|
||||
|
||||
out_tag 为归属地标签时,合并产物命名为「顺心-{out_tag}-实到货物数据.xlsx」,
|
||||
@@ -542,27 +653,27 @@ def shunxin_actual_download_impl(page, out_tag=""):
|
||||
# 2. 读取服务端日期偏移(0=今天,1=昨天…),单日范围:起止同日
|
||||
offset = state_store.get_offset("顺心", "actual")
|
||||
today = datetime.now()
|
||||
target = today - timedelta(days=offset)
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_str = target.strftime("%Y-%m-%d")
|
||||
start_date_str = target_str
|
||||
today_str = target_str
|
||||
print(f">> 正在设置查询日期: [{target_str}](偏移 {offset},0=今天)...")
|
||||
src = f"指定 {date}" if date else f"偏移 {offset},0=今天"
|
||||
print(f">> 正在设置查询日期: [{target_str}]({src})...")
|
||||
|
||||
# 分两步精准呼出和点击时间控件
|
||||
print(" >> 设置起始时间...")
|
||||
page.get_by_placeholder("开始时间").click()
|
||||
page.wait_for_timeout(500)
|
||||
page.locator(
|
||||
f".ant-picker-dropdown:visible td[title='{start_date_str}']"
|
||||
).first.click()
|
||||
_shunxin_pick_date(page, start_date_str)
|
||||
page.wait_for_timeout(300)
|
||||
|
||||
print(" >> 设置截止时间...")
|
||||
page.get_by_placeholder("结束时间").click()
|
||||
page.wait_for_timeout(500)
|
||||
page.locator(
|
||||
f".ant-picker-dropdown:visible td[title='{today_str}']"
|
||||
).first.click()
|
||||
_shunxin_pick_date(page, today_str)
|
||||
page.wait_for_timeout(300)
|
||||
|
||||
# 确认日期
|
||||
|
||||
@@ -11,10 +11,11 @@ from inbound_verify.paths import DOWNLOAD_DIR, CONFIG_PATH
|
||||
from inbound_verify import state_store
|
||||
|
||||
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3, page=None):
|
||||
"""异常兜底:flow 失败 → 重置回初始态 → 重试,最多 max_attempts 次(含首次)。
|
||||
|
||||
每次失败都重置(含最终放弃那一次):既是重试前的清场,也保证最终放弃时环境干净。
|
||||
最后一次失败时(重置前)自动截图保存到 logs/screenshots/,供问题排查。
|
||||
flow 为零参可调用;返回 False 视为失败,其余视为成功。
|
||||
返回 True=最终成功,False=重试耗尽放弃(供调度层判断任务成败)。
|
||||
"""
|
||||
@@ -28,6 +29,13 @@ def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"⚠️ 【{site_name}-{label}】第 {attempt}/{max_attempts} 次失败: {e}")
|
||||
if attempt == max_attempts and page is not None:
|
||||
try:
|
||||
from inbound_verify.runtime import capture_error_screenshot
|
||||
|
||||
capture_error_screenshot(page, site_name, label, attempt, str(e))
|
||||
except Exception:
|
||||
pass
|
||||
print(f" → 重置【{site_name}】到初始态,清理环境 ...")
|
||||
try:
|
||||
reset()
|
||||
@@ -77,6 +85,93 @@ def _remove_if_exists(path):
|
||||
pass
|
||||
|
||||
|
||||
def _yunda_pick_laydate_new(ws_frame, page, date_ymd):
|
||||
"""应到(新版 laydate .layui-laydate):在已打开的面板上选中指定日期(含跨月翻月)。
|
||||
|
||||
目标格子 td[lay-ymd='YYYY-M-D'](非补零)不在当前月视窗时,读 .laydate-set-ym 的
|
||||
当前年月(形如「2026年7月」),按差值点 .laydate-prev-m / .laydate-next-m 翻到目标月,
|
||||
再点格子;同月则直接点。调用前提:#startTime/#endTime 已点开,.layui-laydate:visible 就绪。
|
||||
"""
|
||||
cal = ws_frame.locator(".layui-laydate:visible").first
|
||||
cell = cal.locator(f"td[lay-ymd='{date_ymd}']").first
|
||||
if not cell.is_visible():
|
||||
ty, tm = (int(x) for x in date_ymd.split("-")[:2])
|
||||
print(f" ℹ️ 目标日期 {date_ymd} 不在当前月视窗,正在翻月导航 ...")
|
||||
for _ in range(24):
|
||||
nums = re.findall(r"\d+", cal.locator(".laydate-set-ym").first.inner_text())
|
||||
if len(nums) >= 2:
|
||||
cur_y, cur_m = int(nums[0]), int(nums[1])
|
||||
if cur_y == ty and cur_m == tm:
|
||||
break
|
||||
cur = cur_y * 12 + (cur_m - 1)
|
||||
btn = (
|
||||
".laydate-prev-m"
|
||||
if (ty * 12 + (tm - 1)) < cur
|
||||
else ".laydate-next-m"
|
||||
)
|
||||
cal.locator(btn).first.click()
|
||||
page.wait_for_timeout(300)
|
||||
cell = cal.locator(f"td[lay-ymd='{date_ymd}']").first
|
||||
cell.click()
|
||||
|
||||
|
||||
def _yunda_pick_laydate_old(ws_frame, page, date_ymd):
|
||||
"""实到(旧版 laydate #laydate_box):在已打开的面板上选中指定日期(含跨月翻月)。
|
||||
|
||||
目标格子 td[y][m][d](非补零)不在当前月视窗时,读 #laydate_y/#laydate_m 输入框值
|
||||
(形如「2026年」「07月」)得当前年月,按差值点 #laydate_MM 内 .laydate_chprev /
|
||||
.laydate_chnext 翻到目标月,再点格子;同月则直接点。调用前提:#startDate/#endDate
|
||||
已点开(force=True),#laydate_box:visible 就绪。
|
||||
"""
|
||||
box = ws_frame.locator("#laydate_box:visible").first
|
||||
ty, tm, td = (int(x) for x in date_ymd.split("-")[:3])
|
||||
cell = box.locator(f"td[y='{ty}'][m='{tm}'][d='{td}']").first
|
||||
if not cell.is_visible():
|
||||
print(f" ℹ️ 目标日期 {date_ymd} 不在当前月视窗,正在翻月导航 ...")
|
||||
for _ in range(24):
|
||||
yv = box.locator("#laydate_y").first.evaluate("e=>e.value")
|
||||
mv = box.locator("#laydate_m").first.evaluate("e=>e.value")
|
||||
cur_y = int(re.search(r"\d+", yv).group())
|
||||
cur_m = int(re.search(r"\d+", mv).group())
|
||||
if cur_y == ty and cur_m == tm:
|
||||
break
|
||||
cur = cur_y * 12 + (cur_m - 1)
|
||||
btn = ".laydate_chprev" if (ty * 12 + (tm - 1)) < cur else ".laydate_chnext"
|
||||
box.locator(f"#laydate_MM {btn}").first.click()
|
||||
page.wait_for_timeout(300)
|
||||
cell = box.locator(f"td[y='{ty}'][m='{tm}'][d='{td}']").first
|
||||
cell.click()
|
||||
|
||||
|
||||
def _resolve_export_frame(ws_frame):
|
||||
"""定位韵达数据导出面板内嵌的 iframe。
|
||||
|
||||
韵达改版后导出面板换用 Element UI(全选/向右转移/导出按钮)。实到流程的面板
|
||||
iframe 名为 myFrame(已验证);应到流程历史上为 target1。这里短超时轮流探测,
|
||||
返回首个出现「全选」按钮的 frame;都未命中则 dump 面板内所有 iframe 名便于排查。
|
||||
"""
|
||||
for name in ("myFrame", "target1"):
|
||||
frame = ws_frame.frame_locator(f'iframe[name="{name}"]')
|
||||
try:
|
||||
frame.locator("button.el-button", has_text="全选").first.wait_for(
|
||||
state="visible", timeout=5000
|
||||
)
|
||||
print(f" [导出iframe] 命中 iframe[name={name}]")
|
||||
return frame
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
names = ws_frame.locator("iframe").evaluate_all(
|
||||
"els => els.map(e => e.name || '(无name)')"
|
||||
)
|
||||
print(
|
||||
f" [导出iframe] myFrame/target1 均未命中全选;面板 iframe 名: {names}"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f" [导出iframe] dump 失败: {e}")
|
||||
return ws_frame.frame_locator('iframe[name="myFrame"]')
|
||||
|
||||
|
||||
def yunda_login(page):
|
||||
"""韵达自动登录:未登录则填充表单并提交,已登录则跳过。"""
|
||||
print(">> 正在检查韵达登录状态...")
|
||||
@@ -140,18 +235,19 @@ def yunda_smart_menu_click(page, menu_path):
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
|
||||
def yunda_expected_download(page):
|
||||
def yunda_expected_download(page, force=False, date=None):
|
||||
"""韵达:应到货物数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry(
|
||||
"韵达",
|
||||
"应到",
|
||||
lambda: yunda_expected_download_impl(page),
|
||||
lambda: yunda_expected_download_impl(page, force=force, date=date),
|
||||
lambda: yunda_reset(page),
|
||||
page=page,
|
||||
)
|
||||
|
||||
|
||||
def yunda_expected_download_impl(page):
|
||||
def yunda_expected_download_impl(page, force=False, date=None):
|
||||
"""韵达:应到货物数据下载(单次执行,无重试;供自动化测试探测原始结果用)。"""
|
||||
print("\n▶ 开始执行【韵达 - 应到货物数据下载】任务...")
|
||||
|
||||
@@ -182,11 +278,15 @@ def yunda_expected_download_impl(page):
|
||||
# 2. 读取服务端日期偏移(0=今天,1=昨天…),单日范围:起止同日
|
||||
offset = state_store.get_offset("韵达")
|
||||
today = datetime.now()
|
||||
target = today - timedelta(days=offset)
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
target_ymd = f"{target.year}-{target.month}-{target.day}"
|
||||
start_date_ymd = target_ymd
|
||||
today_ymd = target_ymd
|
||||
print(f">> 设置查询日期: [{target_ymd}](偏移 {offset},0=今天)")
|
||||
src = f"指定 {date}" if date else f"偏移 {offset},0=今天"
|
||||
print(f">> 设置查询日期: [{target_ymd}]({src})")
|
||||
|
||||
# 设定起始时间
|
||||
print(" >> 设置起始时间...")
|
||||
@@ -195,7 +295,7 @@ def yunda_expected_download_impl(page):
|
||||
|
||||
calendar1 = ws_frame.locator(".layui-laydate:visible").first
|
||||
calendar1.wait_for(state="visible", timeout=5000)
|
||||
calendar1.locator(f"td[lay-ymd='{start_date_ymd}']").click()
|
||||
_yunda_pick_laydate_new(ws_frame, page, start_date_ymd)
|
||||
calendar1.locator(".laydate-btns-confirm").click()
|
||||
page.wait_for_timeout(400)
|
||||
|
||||
@@ -205,7 +305,7 @@ def yunda_expected_download_impl(page):
|
||||
|
||||
calendar2 = ws_frame.locator(".layui-laydate:visible").first
|
||||
calendar2.wait_for(state="visible", timeout=5000)
|
||||
calendar2.locator(f"td[lay-ymd='{today_ymd}']").click()
|
||||
_yunda_pick_laydate_new(ws_frame, page, today_ymd)
|
||||
calendar2.locator(".laydate-btns-confirm").click()
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
@@ -254,6 +354,19 @@ def yunda_expected_download_impl(page):
|
||||
row_count = main_rows.count()
|
||||
print(f">> 当前视窗共捕获到活跃交接单记录: {row_count} 条")
|
||||
|
||||
# 【去重】加载本站已落库交接单号;force=True 或查询失败时 existing=空集(不去重)
|
||||
if force:
|
||||
existing = set()
|
||||
print(">> [去重] 强制重下,跳过去重。")
|
||||
else:
|
||||
try:
|
||||
from inbound_verify import store
|
||||
|
||||
existing = store.get_existing_handover_nos("韵达")
|
||||
except Exception as _e:
|
||||
existing = set()
|
||||
print(f">> [去重] 加载已落库交接单号失败,本次不去重: {_e}")
|
||||
|
||||
# 5. 逐行双击并提交导出
|
||||
for i in range(row_count):
|
||||
print(f" ⏳ 正在处理第 {i+1}/{row_count} 个交接单模块...")
|
||||
@@ -261,6 +374,11 @@ def yunda_expected_download_impl(page):
|
||||
|
||||
raw_no = current_row.locator("td").nth(1).inner_text().strip()
|
||||
|
||||
# 【去重】已落库的交接单号不再提交导出任务
|
||||
if raw_no in existing:
|
||||
print(f" ⏭️ 交接单号 {raw_no} 已落库,跳过提交导出。")
|
||||
continue
|
||||
|
||||
# 跳过已绑定的交接单
|
||||
bind_status = current_row.locator("td").nth(2).inner_text().strip()
|
||||
print(f" -> 交接单号: {raw_no} [绑定状态: {bind_status}]")
|
||||
@@ -274,8 +392,11 @@ def yunda_expected_download_impl(page):
|
||||
ws_frame.locator("#docSum").wait_for(state="visible", timeout=15000)
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
# 导出弹窗双层重试:外层重新打开面板,内层重新提交。
|
||||
# 区分字段漏选(补点全选)与字段列表消失(重新打开面板)。
|
||||
# 导出弹窗重试:外层重新打开面板(最多 3 次)。
|
||||
# 韵达站点已将导出面板从 jQuery(.allRight/#submitbutton) 改版为 Element UI
|
||||
# (与实到流程同一导出组件,iframe=myFrame):
|
||||
# 全选(button“全选”) → 向右转移(i.el-icon-d-arrow-right) → 导出(i.el-icon-download)
|
||||
# → 正在导出中(.el-loading-mask) → 成功提示(.el-message-box 导出任务建立成功) → 确定
|
||||
task_success = False
|
||||
for major_attempt in range(3):
|
||||
print(f" >> 正在打开数据导出面板 (尝试 {major_attempt + 1}/3)...")
|
||||
@@ -285,74 +406,61 @@ def yunda_expected_download_impl(page):
|
||||
state="visible", timeout=15000
|
||||
)
|
||||
|
||||
export_frame = ws_frame.frame_locator('iframe[name="target1"]')
|
||||
export_frame = _resolve_export_frame(ws_frame)
|
||||
|
||||
try:
|
||||
# 校验字段列表是否加载完成(以“交接单号”为标志)
|
||||
export_frame.get_by_text("交接单号").first.wait_for(
|
||||
state="visible", timeout=3000
|
||||
)
|
||||
# 校验 Element UI 字段选择区是否加载完成(以「全选」按钮就绪为标志)
|
||||
export_frame.locator(
|
||||
"button.el-button", has_text="全选"
|
||||
).first.wait_for(state="visible", timeout=8000)
|
||||
except Exception:
|
||||
print(" ⚠️ 字段列表未加载,关闭面板后重试...")
|
||||
print(" ⚠️ 导出面板字段区未加载,关闭面板后重试...")
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(1000)
|
||||
continue
|
||||
|
||||
export_frame.locator(".allRight").click()
|
||||
print(" -> 全选字段并向右转移...")
|
||||
export_frame.locator("button.el-button", has_text="全选").first.click()
|
||||
page.wait_for_timeout(400)
|
||||
export_frame.locator(
|
||||
"button.el-button:has(i.el-icon-d-arrow-right)"
|
||||
).first.click()
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
inner_success = False
|
||||
needs_reopen = False
|
||||
|
||||
print(" -> 正在提交导出任务...")
|
||||
for attempt in range(4):
|
||||
export_frame.locator("#submitbutton", has_text="导出数据").click()
|
||||
confirm_link = export_frame.get_by_role("link", name="确定")
|
||||
try:
|
||||
confirm_link.wait_for(state="visible", timeout=6000)
|
||||
if export_frame.get_by_text("导出任务建立成功").is_visible():
|
||||
print(" ✅ 导出任务已建立成功。")
|
||||
confirm_link.click()
|
||||
inner_success = True
|
||||
break
|
||||
elif export_frame.get_by_text(
|
||||
"请选择格式相应的导出字段"
|
||||
).is_visible():
|
||||
confirm_link.click()
|
||||
page.wait_for_timeout(500)
|
||||
export_frame.locator(
|
||||
"button.el-button:has(i.el-icon-download)"
|
||||
).first.click()
|
||||
|
||||
# 区分:字段漏选 还是 字段列表消失
|
||||
if export_frame.get_by_text("交接单号").first.is_visible():
|
||||
print(
|
||||
" ⚠️ 检测到未选择字段(字段列表仍在),重新点击全选..."
|
||||
)
|
||||
export_frame.locator(".allRight").click()
|
||||
page.wait_for_timeout(500)
|
||||
else:
|
||||
print(
|
||||
" ⚠️ 字段列表异常消失,重新打开导出面板..."
|
||||
)
|
||||
needs_reopen = True
|
||||
break # 跳出内层循环,重新打开面板
|
||||
else:
|
||||
confirm_link.click()
|
||||
page.wait_for_timeout(1000)
|
||||
except Exception:
|
||||
page.wait_for_timeout(1000)
|
||||
# 等待「正在导出中」遮罩出现并消失
|
||||
loading = export_frame.locator(".el-loading-mask").first
|
||||
try:
|
||||
loading.wait_for(state="visible", timeout=5000)
|
||||
loading.wait_for(state="hidden", timeout=60000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 等待结果提示并判定
|
||||
inner_success = False
|
||||
try:
|
||||
msg_box = export_frame.locator(".el-message-box.my-alert").first
|
||||
msg_box.wait_for(state="visible", timeout=30000)
|
||||
if msg_box.get_by_text("导出任务建立成功").is_visible():
|
||||
print(" ✅ 导出任务已建立成功。")
|
||||
inner_success = True
|
||||
else:
|
||||
print(" ⚠️ 导出结果提示非成功状态,将重试。")
|
||||
msg_box.locator("button.el-button", has_text="确定").first.click()
|
||||
page.wait_for_timeout(500)
|
||||
except Exception:
|
||||
print(" ⚠️ 未检测到导出结果提示,将重试。")
|
||||
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
if inner_success:
|
||||
task_success = True
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(500)
|
||||
break # 跳出外层循环,继续后续步骤
|
||||
elif needs_reopen:
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(1000)
|
||||
continue # 重新打开面板
|
||||
else:
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(1000)
|
||||
continue
|
||||
break
|
||||
|
||||
if not task_success:
|
||||
raise RuntimeError("多次重试后仍未能建立应到数据离线任务。")
|
||||
@@ -385,18 +493,19 @@ def yunda_expected_download_impl(page):
|
||||
return False
|
||||
|
||||
|
||||
def yunda_actual_download(page):
|
||||
def yunda_actual_download(page, force=False, date=None):
|
||||
"""韵达:实到货物数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry(
|
||||
"韵达",
|
||||
"实到",
|
||||
lambda: yunda_actual_download_impl(page),
|
||||
lambda: yunda_actual_download_impl(page, date=date),
|
||||
lambda: yunda_reset(page),
|
||||
page=page,
|
||||
)
|
||||
|
||||
|
||||
def yunda_actual_download_impl(page):
|
||||
def yunda_actual_download_impl(page, date=None):
|
||||
"""韵达:实到货物数据下载(单次执行,无重试;供自动化测试探测原始结果用)。"""
|
||||
print("\n▶ 开始执行【韵达 - 实到货物数据下载】任务...")
|
||||
|
||||
@@ -427,30 +536,33 @@ def yunda_actual_download_impl(page):
|
||||
|
||||
offset = state_store.get_offset("韵达", "actual")
|
||||
today = datetime.now()
|
||||
target = today - timedelta(days=offset)
|
||||
start_date = target # 单日范围:起止同日
|
||||
today = target # 让下方"截止时间"选择器也指向 target
|
||||
if date:
|
||||
target = datetime.strptime(date, "%Y-%m-%d")
|
||||
else:
|
||||
target = today - timedelta(days=offset)
|
||||
# 旧版 laydate 的日期格子 td[y][m][d] 用非补零整数值;单日范围起止同日
|
||||
target_ymd = f"{target.year}-{target.month}-{target.day}"
|
||||
src = f"指定 {date}" if date else f"偏移 {offset},0=今天"
|
||||
print(
|
||||
f">> 设置实到查询日期: [{target.year}-{target.month}-{target.day}]"
|
||||
f"(偏移 {offset},0=今天)"
|
||||
f">> 设置实到查询日期: [{target.year}-{target.month}-{target.day}]({src})"
|
||||
)
|
||||
|
||||
print(" >> 正在设定起始时间...")
|
||||
ws_frame.locator("#startDate").click()
|
||||
page.wait_for_timeout(400)
|
||||
box1 = ws_frame.locator("#laydate_box:visible").first
|
||||
box1.locator(
|
||||
f"td[y='{start_date.year}'][m='{start_date.month}'][d='{start_date.day}']"
|
||||
).click()
|
||||
ws_frame.locator("#laydate_box:visible").first.wait_for(
|
||||
state="visible", timeout=5000
|
||||
)
|
||||
_yunda_pick_laydate_old(ws_frame, page, target_ymd)
|
||||
page.wait_for_timeout(400)
|
||||
|
||||
print(" >> 正在设定截止时间...")
|
||||
ws_frame.locator("#endDate").click()
|
||||
page.wait_for_timeout(400)
|
||||
box2 = ws_frame.locator("#laydate_box:visible").first
|
||||
box2.locator(
|
||||
f"td[y='{today.year}'][m='{today.month}'][d='{today.day}']"
|
||||
).click()
|
||||
ws_frame.locator("#laydate_box:visible").first.wait_for(
|
||||
state="visible", timeout=5000
|
||||
)
|
||||
_yunda_pick_laydate_old(ws_frame, page, target_ymd)
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
print(" >> 正在变更扫描类型为【到件】...")
|
||||
@@ -497,8 +609,10 @@ def yunda_actual_download_impl(page):
|
||||
).click()
|
||||
return
|
||||
|
||||
# 导出弹窗双层重试:外层重新打开面板,内层重新提交。
|
||||
# 区分字段漏选(补点全选)与字段列表消失(重新打开面板)。
|
||||
# 导出弹窗重试:外层重新打开面板(最多 3 次)。
|
||||
# 韵达站点已将导出面板从 jQuery(.allRight/#submitbutton) 改版为 Element UI:
|
||||
# 全选(button“全选”) → 向右转移(i.el-icon-d-arrow-right) → 导出(i.el-icon-download)
|
||||
# → 正在导出中(.el-loading-mask) → 成功提示(.el-message-box 导出任务建立成功) → 确定
|
||||
print(">> 正在发起导出...")
|
||||
task_success = False
|
||||
|
||||
@@ -512,69 +626,58 @@ def yunda_actual_download_impl(page):
|
||||
export_frame = ws_frame.frame_locator('iframe[name="myFrame"]')
|
||||
|
||||
try:
|
||||
# 校验字段列表是否加载完成(以“扫描类型”为标志)
|
||||
export_frame.get_by_text("扫描类型").first.wait_for(
|
||||
state="visible", timeout=3000
|
||||
)
|
||||
# 校验 Element UI 字段选择区是否加载完成(以「全选」按钮就绪为标志)
|
||||
export_frame.locator(
|
||||
"button.el-button", has_text="全选"
|
||||
).first.wait_for(state="visible", timeout=8000)
|
||||
except Exception:
|
||||
print(" ⚠️ 字段列表未加载,关闭面板后重试...")
|
||||
print(" ⚠️ 导出面板字段区未加载,关闭面板后重试...")
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(1000)
|
||||
continue
|
||||
|
||||
export_frame.locator(".allRight").click()
|
||||
print(" -> 全选字段并向右转移...")
|
||||
export_frame.locator("button.el-button", has_text="全选").first.click()
|
||||
page.wait_for_timeout(400)
|
||||
export_frame.locator(
|
||||
"button.el-button:has(i.el-icon-d-arrow-right)"
|
||||
).first.click()
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
inner_success = False
|
||||
needs_reopen = False
|
||||
|
||||
print(" -> 正在提交导出任务...")
|
||||
for attempt in range(4):
|
||||
export_frame.locator("#submitbutton", has_text="导出数据").click()
|
||||
confirm_link = export_frame.get_by_role("link", name="确定")
|
||||
try:
|
||||
confirm_link.wait_for(state="visible", timeout=6000)
|
||||
if export_frame.get_by_text("导出任务建立成功").is_visible():
|
||||
print(" ✅ 导出任务已建立成功。")
|
||||
confirm_link.click()
|
||||
inner_success = True
|
||||
break
|
||||
elif export_frame.get_by_text(
|
||||
"请选择格式相应的导出字段"
|
||||
).is_visible():
|
||||
confirm_link.click()
|
||||
page.wait_for_timeout(500)
|
||||
export_frame.locator(
|
||||
"button.el-button:has(i.el-icon-download)"
|
||||
).first.click()
|
||||
|
||||
# 区分:字段漏选 还是 字段列表消失
|
||||
if export_frame.get_by_text("扫描类型").first.is_visible():
|
||||
print(
|
||||
" ⚠️ 检测到未选择字段(字段列表仍在),重新点击全选..."
|
||||
)
|
||||
export_frame.locator(".allRight").click()
|
||||
page.wait_for_timeout(500)
|
||||
else:
|
||||
print(" ⚠️ 字段列表异常消失,重新打开导出面板...")
|
||||
needs_reopen = True
|
||||
break
|
||||
else:
|
||||
confirm_link.click()
|
||||
page.wait_for_timeout(1000)
|
||||
except Exception:
|
||||
page.wait_for_timeout(1000)
|
||||
# 等待「正在导出中」遮罩出现并消失
|
||||
loading = export_frame.locator(".el-loading-mask").first
|
||||
try:
|
||||
loading.wait_for(state="visible", timeout=5000)
|
||||
loading.wait_for(state="hidden", timeout=60000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 等待结果提示并判定
|
||||
inner_success = False
|
||||
try:
|
||||
msg_box = export_frame.locator(".el-message-box.my-alert").first
|
||||
msg_box.wait_for(state="visible", timeout=30000)
|
||||
if msg_box.get_by_text("导出任务建立成功").is_visible():
|
||||
print(" ✅ 导出任务已建立成功。")
|
||||
inner_success = True
|
||||
else:
|
||||
print(" ⚠️ 导出结果提示非成功状态,将重试。")
|
||||
msg_box.locator("button.el-button", has_text="确定").first.click()
|
||||
page.wait_for_timeout(500)
|
||||
except Exception:
|
||||
print(" ⚠️ 未检测到导出结果提示,将重试。")
|
||||
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
if inner_success:
|
||||
task_success = True
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(500)
|
||||
break
|
||||
elif needs_reopen:
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(1000)
|
||||
continue
|
||||
else:
|
||||
ws_frame.locator(".layui-layer-close1").click()
|
||||
page.wait_for_timeout(1000)
|
||||
continue
|
||||
|
||||
if not task_success:
|
||||
raise RuntimeError("多次重试后仍未能建立实到数据离线任务。")
|
||||
|
||||
@@ -4,17 +4,18 @@ import os
|
||||
import re
|
||||
import time
|
||||
import yaml
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
import pandas as pd
|
||||
|
||||
from inbound_verify.paths import DOWNLOAD_DIR, CONFIG_PATH
|
||||
from inbound_verify import state_store
|
||||
|
||||
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
def with_retry(site_name, label, flow, reset, max_attempts=3, page=None):
|
||||
"""异常兜底:flow 失败 → 重置回初始态 → 重试,最多 max_attempts 次(含首次)。
|
||||
|
||||
每次失败都重置(含最终放弃那一次):既是重试前的清场,也保证最终放弃时环境干净。
|
||||
最后一次失败时(重置前)自动截图保存到 logs/screenshots/,供问题排查。
|
||||
flow 为零参可调用;返回 False 视为失败,其余视为成功。
|
||||
返回 True=最终成功,False=重试耗尽放弃(供调度层判断任务成败)。
|
||||
"""
|
||||
@@ -28,6 +29,13 @@ def with_retry(site_name, label, flow, reset, max_attempts=3):
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"⚠️ 【{site_name}-{label}】第 {attempt}/{max_attempts} 次失败: {e}")
|
||||
if attempt == max_attempts and page is not None:
|
||||
try:
|
||||
from inbound_verify.runtime import capture_error_screenshot
|
||||
|
||||
capture_error_screenshot(page, site_name, label, attempt, str(e))
|
||||
except Exception:
|
||||
pass
|
||||
print(f" → 重置【{site_name}】到初始态,清理环境 ...")
|
||||
try:
|
||||
reset()
|
||||
@@ -93,6 +101,52 @@ def _dom_click(locator):
|
||||
)
|
||||
|
||||
|
||||
def _zto_compute_target_time(offset):
|
||||
"""直接从 Python datetime 计算目标日期的毫秒级时间戳(本地时区零点)。
|
||||
|
||||
不再从 DOM 的 real-today 元素读取 time 属性,避免双月视图下 real-today
|
||||
同时出现在 month1(隐藏 ghost cell)和 month2(可见)导致 .first 取到隐藏元素。
|
||||
"""
|
||||
target_date = datetime.now().date() - timedelta(days=offset)
|
||||
target_dt = datetime(target_date.year, target_date.month, target_date.day)
|
||||
return int(target_dt.timestamp() * 1000)
|
||||
|
||||
|
||||
def _zto_find_visible_day(frame_locator, target_time):
|
||||
"""在双月日期控件中查找可见的日期格子。
|
||||
|
||||
jQuery Date Range Picker 双月视图下,同一天可能出现在两个面板中:
|
||||
- month1(左面板)的溢出 ghost cell:display:none,不可见
|
||||
- month2(右面板)的正常 cell:可见
|
||||
|
||||
同一日期在 DOM 中可能有毫秒级差异(零点 vs 23:59:59),遍历匹配并返回
|
||||
第一个 visible 的;无可见匹配返回 None。
|
||||
"""
|
||||
# 尝试两个时间变体:零点 和 23:59:59(部分 checked/selected 格用后者)
|
||||
for time_variant in (target_time, target_time + 86399000):
|
||||
sel = f"td div.day[time='{time_variant}']"
|
||||
cells = frame_locator.locator(sel)
|
||||
count = cells.count()
|
||||
for i in range(count):
|
||||
if cells.nth(i).is_visible():
|
||||
return cells.nth(i)
|
||||
return None
|
||||
|
||||
|
||||
def _zto_flip_to_target_month(frame_locator, page, target_time, max_flips=12):
|
||||
"""中通日历(jQuery-Date-Range-Picker 双月视图)跨月导航:目标日期不在当前视窗时,
|
||||
循环点 .prev 把目标月翻进视窗。用 _zto_find_visible_day 判可见(跳过隐藏 ghost cell)。
|
||||
返回 True 若目标格子最终可见。"""
|
||||
for _ in range(max_flips):
|
||||
if _zto_find_visible_day(frame_locator, target_time) is not None:
|
||||
return True
|
||||
frame_locator.locator(".date-picker-wrapper .prev").first.evaluate(
|
||||
"el => el.click()"
|
||||
)
|
||||
page.wait_for_timeout(450)
|
||||
return _zto_find_visible_day(frame_locator, target_time) is not None
|
||||
|
||||
|
||||
def zto_smart_menu_click(page, menu_path):
|
||||
"""中通菜单导航"""
|
||||
print(f">> 正在导航: {' -> '.join(menu_path)}")
|
||||
@@ -109,18 +163,19 @@ def zto_smart_menu_click(page, menu_path):
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
|
||||
def zto_expected_download(page):
|
||||
def zto_expected_download(page, force=False, date=None):
|
||||
"""中通:应到货物数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry(
|
||||
"中通",
|
||||
"应到",
|
||||
lambda: zto_expected_download_impl(page),
|
||||
lambda: zto_expected_download_impl(page, force=force, date=date),
|
||||
lambda: zto_reset(page),
|
||||
page=page,
|
||||
)
|
||||
|
||||
|
||||
def zto_expected_download_impl(page):
|
||||
def zto_expected_download_impl(page, force=False, date=None):
|
||||
"""中通:应到货物数据下载(单次执行,无重试;供自动化测试探测原始结果用)。"""
|
||||
print("\n▶ 开始执行【中通 - 应到货物数据下载】任务...")
|
||||
|
||||
@@ -144,33 +199,35 @@ def zto_expected_download_impl(page):
|
||||
|
||||
# 读取服务端日期偏移(0=今天,1=昨天…),单日:起止同日
|
||||
offset = state_store.get_offset("中通")
|
||||
|
||||
print(f">> 正在设定查询日期: 偏移 {offset}(0=今天)...")
|
||||
if date:
|
||||
# 指定日期:折算成相对今天的有效偏移,复用下方 target_time 计算与跨月翻月
|
||||
target_date = datetime.strptime(date, "%Y-%m-%d").date()
|
||||
offset = (datetime.now().date() - target_date).days
|
||||
print(f">> 正在设定查询日期: 指定日期 {date}(折算偏移 {offset})...")
|
||||
else:
|
||||
print(f">> 正在设定查询日期: 偏移 {offset}(0=今天)...")
|
||||
ewb_frame.locator("#beginDate").click()
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
today_cell = ewb_frame.locator("td div.day.real-today").first
|
||||
today_cell.wait_for(state="visible")
|
||||
# 直接从 Python datetime 计算目标时间戳,不再依赖 DOM real-today(双月视图
|
||||
# 下 real-today 可能同时出现在 month1 隐藏 ghost cell 和 month2 可见 cell,
|
||||
# .first 会取到隐藏的那个导致 wait_for(visible) 超时)。
|
||||
target_time = _zto_compute_target_time(offset)
|
||||
target_cell = _zto_find_visible_day(ewb_frame, target_time)
|
||||
|
||||
today_time_str = today_cell.get_attribute("time")
|
||||
if today_time_str:
|
||||
today_time = int(today_time_str)
|
||||
target_time = today_time - offset * 86400000
|
||||
target_cell = ewb_frame.locator(f"td div.day[time='{target_time}']").first
|
||||
if target_cell is None:
|
||||
print(" ℹ️ 目标日期不在当前视窗,正在翻月导航 ...")
|
||||
if not _zto_flip_to_target_month(ewb_frame, page, target_time):
|
||||
raise RuntimeError(f"翻月后仍无法定位目标日期格子(time={target_time})")
|
||||
target_cell = _zto_find_visible_day(ewb_frame, target_time)
|
||||
if target_cell is None:
|
||||
raise RuntimeError(f"翻月后仍无法定位目标日期格子(time={target_time})")
|
||||
|
||||
if target_cell.is_visible():
|
||||
target_cell.click()
|
||||
page.wait_for_timeout(300)
|
||||
target_cell.click()
|
||||
else:
|
||||
print(" ⚠️ 偏移日期不在当前日历视窗内,自动降级为查询当天。")
|
||||
today_cell.click()
|
||||
page.wait_for_timeout(300)
|
||||
today_cell.click()
|
||||
else:
|
||||
today_cell.click()
|
||||
page.wait_for_timeout(300)
|
||||
today_cell.click()
|
||||
# 日期格子用 _dom_click 直接派发事件:.click() 会先 hover 格子,触发
|
||||
# "范围长度"提示气泡(.date-range-length-tip)盖住格子导致点击被遮挡超时。
|
||||
_dom_click(target_cell)
|
||||
page.wait_for_timeout(300)
|
||||
_dom_click(target_cell)
|
||||
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
@@ -240,6 +297,19 @@ def zto_expected_download_impl(page):
|
||||
count = main_rows.count()
|
||||
print(f">> 共发现 {count} 个交接单需要导出。")
|
||||
|
||||
# 【去重】加载本站已落库交接单号;force=True 或查询失败时 existing=空集(不去重)
|
||||
if force:
|
||||
existing = set()
|
||||
print(">> [去重] 强制重下,跳过去重。")
|
||||
else:
|
||||
try:
|
||||
from inbound_verify import store
|
||||
|
||||
existing = store.get_existing_handover_nos("中通")
|
||||
except Exception as _e:
|
||||
existing = set()
|
||||
print(f">> [去重] 加载已落库交接单号失败,本次不去重: {_e}")
|
||||
|
||||
for i in range(count):
|
||||
print(f" ⏳ 正在处理第 {i+1}/{count} 个交接单...")
|
||||
row = ewb_frame.locator(
|
||||
@@ -251,6 +321,11 @@ def zto_expected_download_impl(page):
|
||||
handover_no = match.group(0) if match else raw_text.strip()
|
||||
print(f" -> 当前交接单号:{handover_no}")
|
||||
|
||||
# 【去重】已落库的交接单号不再提交导出任务(不双击、不 append export_times)
|
||||
if handover_no in existing:
|
||||
print(f" ⏭️ 交接单号 {handover_no} 已落库,跳过提交导出。")
|
||||
continue
|
||||
|
||||
row.dblclick()
|
||||
|
||||
ewb_frame.locator("#datagrid2").get_by_text("运单号").wait_for(
|
||||
@@ -316,6 +391,11 @@ def zto_expected_download_impl(page):
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 关闭【进站交接单查询】标签页时出错: {e}")
|
||||
|
||||
# 【去重兜底】全部已落库/无数据 → 无导出任务,标签页已关,直接结束不进轮询
|
||||
if not export_times:
|
||||
print(">> 本次无新交接单需导出(全部已落库或无数据),结束。")
|
||||
return True
|
||||
|
||||
# 交由统一的轮询下载流程处理
|
||||
_zto_poll_and_download_tasks(
|
||||
page,
|
||||
@@ -330,18 +410,19 @@ def zto_expected_download_impl(page):
|
||||
return False
|
||||
|
||||
|
||||
def zto_actual_download(page):
|
||||
def zto_actual_download(page, force=False, date=None):
|
||||
"""中通:实到货物数据下载(内部含异常兜底重试,路由层无感)。"""
|
||||
|
||||
return with_retry(
|
||||
"中通",
|
||||
"实到",
|
||||
lambda: zto_actual_download_impl(page),
|
||||
lambda: zto_actual_download_impl(page, date=date),
|
||||
lambda: zto_reset(page),
|
||||
page=page,
|
||||
)
|
||||
|
||||
|
||||
def zto_actual_download_impl(page):
|
||||
def zto_actual_download_impl(page, date=None):
|
||||
"""中通:实到货物数据下载(单次执行,无重试;供自动化测试探测原始结果用)。"""
|
||||
print("\n▶ 开始执行【中通 - 实到货物数据下载】任务...")
|
||||
|
||||
@@ -365,34 +446,34 @@ def zto_actual_download_impl(page):
|
||||
|
||||
# 2. 读取服务端日期偏移(0=今天,1=昨天…),单日:起止同日
|
||||
offset = state_store.get_offset("中通", "actual")
|
||||
|
||||
print(f">> 正在设定查询日期: 偏移 {offset}(0=今天)...")
|
||||
if date:
|
||||
target_date = datetime.strptime(date, "%Y-%m-%d").date()
|
||||
offset = (datetime.now().date() - target_date).days
|
||||
print(f">> 正在设定查询日期: 指定日期 {date}(折算偏移 {offset})...")
|
||||
else:
|
||||
print(f">> 正在设定查询日期: 偏移 {offset}(0=今天)...")
|
||||
arr_frame.locator("#daterange").click()
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
today_cell = arr_frame.locator("td div.day.real-today").first
|
||||
today_cell.wait_for(state="visible")
|
||||
# 直接从 Python datetime 计算目标时间戳,不再依赖 DOM real-today(双月视图
|
||||
# 下 real-today 可能同时出现在 month1 隐藏 ghost cell 和 month2 可见 cell,
|
||||
# .first 会取到隐藏的那个导致 wait_for(visible) 超时)。
|
||||
target_time = _zto_compute_target_time(offset)
|
||||
target_cell = _zto_find_visible_day(arr_frame, target_time)
|
||||
|
||||
today_time_str = today_cell.get_attribute("time")
|
||||
if today_time_str:
|
||||
today_time = int(today_time_str)
|
||||
target_time = today_time - offset * 86400000
|
||||
target_cell = arr_frame.locator(f"td div.day[time='{target_time}']").first
|
||||
if target_cell is None:
|
||||
print(" ℹ️ 目标日期不在当前视窗,正在翻月导航 ...")
|
||||
if not _zto_flip_to_target_month(arr_frame, page, target_time):
|
||||
raise RuntimeError(f"翻月后仍无法定位目标日期格子(time={target_time})")
|
||||
target_cell = _zto_find_visible_day(arr_frame, target_time)
|
||||
if target_cell is None:
|
||||
raise RuntimeError(f"翻月后仍无法定位目标日期格子(time={target_time})")
|
||||
|
||||
# 日期格子用 _dom_click 直接派发事件:Playwright 的 .click() 会先 hover 格子,
|
||||
# 触发“范围长度”提示气泡(.date-range-length-tip)盖住格子,导致点击被判遮挡而超时。
|
||||
if target_cell.is_visible():
|
||||
_dom_click(target_cell)
|
||||
page.wait_for_timeout(300)
|
||||
_dom_click(target_cell)
|
||||
else:
|
||||
_dom_click(today_cell)
|
||||
page.wait_for_timeout(300)
|
||||
_dom_click(today_cell)
|
||||
else:
|
||||
_dom_click(today_cell)
|
||||
page.wait_for_timeout(300)
|
||||
_dom_click(today_cell)
|
||||
# 日期格子用 _dom_click 直接派发事件:Playwright 的 .click() 会先 hover 格子,
|
||||
# 触发"范围长度"提示气泡(.date-range-length-tip)盖住格子,导致点击被判遮挡而超时。
|
||||
_dom_click(target_cell)
|
||||
page.wait_for_timeout(300)
|
||||
_dom_click(target_cell)
|
||||
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
@@ -641,7 +722,7 @@ def _zto_poll_and_download_tasks(page, export_times, download_dir, final_filenam
|
||||
)
|
||||
|
||||
# ====================================================================
|
||||
# 所有目标文件下载完成后,关闭“导出任务管理”标签页
|
||||
# 所有目标文件下载完成后,关闭"导出任务管理"标签页
|
||||
# ====================================================================
|
||||
print(">> 【导出任务管理】下载完成,正在关闭标签页...")
|
||||
try:
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from inbound_verify.paths import STATE_DB_PATH
|
||||
|
||||
@@ -32,7 +32,7 @@ def _now():
|
||||
def init_db():
|
||||
"""建库建表(幂等)。确保 state 目录存在。"""
|
||||
os.makedirs(os.path.dirname(STATE_DB_PATH), exist_ok=True)
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS site_status (
|
||||
site TEXT PRIMARY KEY,
|
||||
@@ -70,9 +70,22 @@ def init_db():
|
||||
status TEXT,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
error TEXT
|
||||
error TEXT,
|
||||
trigger TEXT NOT NULL DEFAULT '',
|
||||
target_date TEXT NOT NULL DEFAULT '',
|
||||
force INTEGER NOT NULL DEFAULT 0
|
||||
)
|
||||
""")
|
||||
# 旧库迁移:补触发方式/目标日期/强制重下三列(新库已含;重复添加抛 OperationalError,忽略)
|
||||
for _col, _typedef in [
|
||||
("trigger", "TEXT NOT NULL DEFAULT ''"),
|
||||
("target_date", "TEXT NOT NULL DEFAULT ''"),
|
||||
("force", "INTEGER NOT NULL DEFAULT 0"),
|
||||
]:
|
||||
try:
|
||||
conn.execute(f"ALTER TABLE task_history ADD COLUMN {_col} {_typedef}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS site_config (
|
||||
site TEXT PRIMARY KEY,
|
||||
@@ -121,6 +134,21 @@ def init_db():
|
||||
PRIMARY KEY (site, kind)
|
||||
)
|
||||
""")
|
||||
# 周期性抓取调度(每 site×kind 一行):取代旧 site_config.schedule_* 的每日单时点。
|
||||
# enabled=总开关;active_start/end=激活时段"HH:MM"(空=不限时段,避免半夜空跑);
|
||||
# interval_minutes=激活时段内的抓取间隔。百世 kind 固定为 undelivered(无应到/实到二分)。
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS fetch_schedule (
|
||||
site TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 0,
|
||||
active_start TEXT NOT NULL DEFAULT '',
|
||||
active_end TEXT NOT NULL DEFAULT '',
|
||||
interval_minutes INTEGER NOT NULL DEFAULT 30,
|
||||
updated_at TEXT,
|
||||
PRIMARY KEY (site, kind)
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
@@ -202,14 +230,14 @@ def _upsert(conn, site, **fields):
|
||||
def set_login_state(site, logged_in):
|
||||
"""更新单站登录态。logged_in: bool。"""
|
||||
state = LOGIN_IN if logged_in else LOGIN_OUT
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
_upsert(conn, site, login_state=state, login_checked_at=_now())
|
||||
|
||||
|
||||
def reset_login_states(sites):
|
||||
"""启动时把给定站点的登录态重置为 unknown(避免显示上一会话的陈旧登录态)。
|
||||
登录态是会话级的;数据态(文件就绪)会话无关、保留不动,心跳就绪后会重新探测。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
for site in sites:
|
||||
_upsert(conn, site, login_state=LOGIN_UNKNOWN, login_checked_at="")
|
||||
|
||||
@@ -223,15 +251,34 @@ def set_data_state(site, kind, ready, generated_at, business_date=None):
|
||||
}
|
||||
if business_date is not None:
|
||||
fields[f"{kind}_business_date"] = business_date or ""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
_upsert(conn, site, **fields)
|
||||
|
||||
|
||||
def set_business_date(site, kind, business_date):
|
||||
"""仅写业务日期快照(不碰 ready/generated_at)。
|
||||
|
||||
下载成功钩子用:ready 语义已移交「入库成功」(见 reset_data_ready / _persist_to_db),
|
||||
下载阶段只记业务日期,供前端状态盘显示「是哪天的数据」。
|
||||
"""
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
_upsert(conn, site, **{f"{kind}_business_date": business_date or ""})
|
||||
|
||||
|
||||
def set_ready(site, kind, ready):
|
||||
"""仅写就绪态(不碰 business_date/generated_at)。
|
||||
|
||||
供心跳从 ingest_state 派生 ready 用——ready 现为 DB 入库真相的派生视图,
|
||||
非启动重置、不读 Excel。"""
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
_upsert(conn, site, **{f"{kind}_ready": 1 if ready else 0})
|
||||
|
||||
|
||||
def get_all_status():
|
||||
"""返回 {site: {各字段}};库不存在则返回 {}。"""
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return {}
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT site, login_state, login_checked_at, expected_ready, "
|
||||
"expected_generated_at, expected_business_date, actual_ready, "
|
||||
@@ -260,7 +307,7 @@ def get_all_status():
|
||||
|
||||
def set_ingest_state(site, kind, ok, count=0, error=None):
|
||||
"""记录一次入库结果(UPSERT)。ok: bool;count: 入库条数;error: 失败原因或 None。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO ingest_state (site, kind, ok, ingested_at, count, error) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?) "
|
||||
@@ -276,7 +323,7 @@ def get_all_ingest_state():
|
||||
"""返回 {site: {kind: {ok, ingested_at, count, error}}};库不存在返回 {}。"""
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return {}
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT site, kind, ok, ingested_at, count, error FROM ingest_state"
|
||||
).fetchall()
|
||||
@@ -301,7 +348,7 @@ def get_offset(site, kind="expected"):
|
||||
col = "expected_offset" if kind == "expected" else "actual_offset"
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return 0
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
row = conn.execute(
|
||||
f"SELECT {col} FROM site_config WHERE site=?", (site,)
|
||||
).fetchone()
|
||||
@@ -312,7 +359,7 @@ def set_offset(site, kind, offset):
|
||||
"""设置单站下载日期偏移(kind: 'expected'/'actual'),钳制到 [0, MAX_DATE_OFFSET]。"""
|
||||
col = "expected_offset" if kind == "expected" else "actual_offset"
|
||||
offset = max(0, min(MAX_DATE_OFFSET, int(offset)))
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO site_config (site, {col}, updated_at) VALUES (?, ?, ?) "
|
||||
f"ON CONFLICT(site) DO UPDATE SET {col}=excluded.{col}, "
|
||||
@@ -323,10 +370,32 @@ def set_offset(site, kind, offset):
|
||||
return offset
|
||||
|
||||
|
||||
def resolve_target_date(site, kind, date=None):
|
||||
"""计算一条任务的目标下载日期(YYYY-MM-DD,供任务日志展示 / 重试回放)。
|
||||
有 date 用 date;否则按站点偏移推算(与 runtime._record_business_date 同源):
|
||||
expected → 应到偏移;actual → 实到偏移;百世 undelivered → 当天;
|
||||
4 站 undelivered → 跟随应到偏移。__compare__ 无数据概念,返回 ''。"""
|
||||
if site == "__compare__":
|
||||
return ""
|
||||
if date:
|
||||
return date
|
||||
today = datetime.now().date()
|
||||
if kind == "expected":
|
||||
return (today - timedelta(days=get_offset(site, "expected"))).strftime(
|
||||
"%Y-%m-%d"
|
||||
)
|
||||
if kind == "actual":
|
||||
return (today - timedelta(days=get_offset(site, "actual"))).strftime("%Y-%m-%d")
|
||||
if site == "百世":
|
||||
return today.strftime("%Y-%m-%d")
|
||||
return (today - timedelta(days=get_offset(site, "expected"))).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def set_schedule(site, enabled, time_str):
|
||||
"""设置单站每日定时下载(enabled: bool;time_str: 'HH:MM' 或 '')。"""
|
||||
"""【DEPRECATED】旧"每日单时点定时"——已被 fetch_schedule 的周期+激活时段模式取代。
|
||||
保留死代码以防外部残留调用;新代码请用 set_fetch_schedule。"""
|
||||
enabled_int = 1 if enabled else 0
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO site_config (site, schedule_enabled, schedule_time, updated_at) "
|
||||
"VALUES (?, ?, ?, ?) "
|
||||
@@ -339,23 +408,128 @@ def set_schedule(site, enabled, time_str):
|
||||
return bool(enabled_int), (time_str or "")
|
||||
|
||||
|
||||
def get_all_config():
|
||||
"""返回 {site: {expected_offset, actual_offset, schedule_enabled, schedule_time}}。"""
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return {}
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT site, expected_offset, actual_offset, schedule_enabled, schedule_time "
|
||||
"FROM site_config"
|
||||
).fetchall()
|
||||
# ============================ 周期性抓取调度(fetch_schedule)============================
|
||||
# 取代旧 site_config.schedule_* 的"每日单时点":每 site×kind 一行,
|
||||
# 在激活时段 [active_start, active_end) 内按 interval_minutes 周期抓取。
|
||||
# 应到/实到各自独立配置;百世只有 undelivered(站点直供未到明细,无应到/实到二分)。
|
||||
|
||||
# 各站允许的周期抓取 kind(单一来源,server.py 复用)
|
||||
SITE_FETCH_KINDS = {
|
||||
"顺心": ("expected", "actual"),
|
||||
"中通": ("expected", "actual"),
|
||||
"韵达": ("expected", "actual"),
|
||||
"安能": ("expected", "actual"),
|
||||
"百世": ("undelivered",),
|
||||
}
|
||||
|
||||
DEFAULT_FETCH_SCHEDULE = {
|
||||
"enabled": False,
|
||||
"active_start": "",
|
||||
"active_end": "",
|
||||
"interval_minutes": 30,
|
||||
}
|
||||
|
||||
|
||||
def allowed_kinds(site):
|
||||
"""该站允许的周期抓取 kind 元组;未知站点返回空元组。"""
|
||||
return SITE_FETCH_KINDS.get(site, ())
|
||||
|
||||
|
||||
def _fetch_spec(row):
|
||||
"""把 fetch_schedule 行转成 spec dict。"""
|
||||
return {
|
||||
r[0]: {
|
||||
"expected_offset": int(r[1]),
|
||||
"actual_offset": int(r[2]),
|
||||
"schedule_enabled": bool(r[3]),
|
||||
"schedule_time": r[4] or "",
|
||||
"enabled": bool(row[0]),
|
||||
"active_start": row[1] or "",
|
||||
"active_end": row[2] or "",
|
||||
"interval_minutes": int(row[3]),
|
||||
}
|
||||
|
||||
|
||||
def get_fetch_schedule(site, kind):
|
||||
"""读单 (site,kind) 周期抓取配置;未配置返回 None。"""
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return None
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT enabled, active_start, active_end, interval_minutes "
|
||||
"FROM fetch_schedule WHERE site=? AND kind=?",
|
||||
(site, kind),
|
||||
).fetchone()
|
||||
return _fetch_spec(row) if row else None
|
||||
|
||||
|
||||
def set_fetch_schedule(site, kind, enabled, active_start, active_end, interval_minutes):
|
||||
"""UPSERT 单 (site,kind) 周期抓取配置;返回写入后的 spec dict。"""
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO fetch_schedule "
|
||||
"(site, kind, enabled, active_start, active_end, interval_minutes, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(site, kind) DO UPDATE SET "
|
||||
"enabled=excluded.enabled, active_start=excluded.active_start, "
|
||||
"active_end=excluded.active_end, interval_minutes=excluded.interval_minutes, "
|
||||
"updated_at=excluded.updated_at",
|
||||
(
|
||||
site,
|
||||
kind,
|
||||
1 if enabled else 0,
|
||||
(active_start or ""),
|
||||
(active_end or ""),
|
||||
int(interval_minutes),
|
||||
_now(),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return {
|
||||
"enabled": bool(enabled),
|
||||
"active_start": active_start or "",
|
||||
"active_end": active_end or "",
|
||||
"interval_minutes": int(interval_minutes),
|
||||
}
|
||||
|
||||
|
||||
def get_all_fetch_schedules():
|
||||
"""返回 {site: {kind: spec}};对每个站点的每个 allowed kind 都补齐(缺失用默认值)。
|
||||
保证前端永远拿到完整 kind 集,不必前端补默认 spec。"""
|
||||
out = {}
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
for site, kinds in SITE_FETCH_KINDS.items():
|
||||
out[site] = {k: dict(DEFAULT_FETCH_SCHEDULE) for k in kinds}
|
||||
return out
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT site, kind, enabled, active_start, active_end, interval_minutes "
|
||||
"FROM fetch_schedule"
|
||||
).fetchall()
|
||||
by_key = {(r[0], r[1]): _fetch_spec(r[2:]) for r in rows}
|
||||
for site, kinds in SITE_FETCH_KINDS.items():
|
||||
out[site] = {
|
||||
k: by_key.get((site, k), dict(DEFAULT_FETCH_SCHEDULE)) for k in kinds
|
||||
}
|
||||
for r in rows
|
||||
return out
|
||||
|
||||
|
||||
def get_all_config():
|
||||
"""返回 {site: {expected_offset, actual_offset, fetch_schedules}}。
|
||||
站点集以 fetch_schedule 的 allowed kinds 为准(覆盖全业务站点);offsets 缺失默认 0。"""
|
||||
schedules = get_all_fetch_schedules()
|
||||
offsets = {}
|
||||
if os.path.exists(STATE_DB_PATH):
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT site, expected_offset, actual_offset FROM site_config"
|
||||
).fetchall()
|
||||
offsets = {
|
||||
r[0]: {"expected_offset": int(r[1]), "actual_offset": int(r[2])}
|
||||
for r in rows
|
||||
}
|
||||
return {
|
||||
site: {
|
||||
"expected_offset": offsets.get(site, {}).get("expected_offset", 0),
|
||||
"actual_offset": offsets.get(site, {}).get("actual_offset", 0),
|
||||
"fetch_schedules": schedules.get(site, {}),
|
||||
}
|
||||
for site in schedules
|
||||
}
|
||||
|
||||
|
||||
@@ -367,7 +541,7 @@ def get_setting(site, key):
|
||||
"""读取单站某个配置值;未设置返回 ''。"""
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return ""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT value FROM site_settings WHERE site=? AND key=?", (site, key)
|
||||
).fetchone()
|
||||
@@ -376,7 +550,7 @@ def get_setting(site, key):
|
||||
|
||||
def set_setting(site, key, value):
|
||||
"""设置单站某个配置值(upsert)。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO site_settings (site, key, value) VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(site, key) DO UPDATE SET value=excluded.value",
|
||||
@@ -389,7 +563,7 @@ def get_site_settings(site):
|
||||
"""返回单站全部配置 {key: value}。"""
|
||||
if not os.path.exists(STATE_DB_PATH):
|
||||
return {}
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT key, value FROM site_settings WHERE site=?", (site,)
|
||||
).fetchall()
|
||||
@@ -399,14 +573,42 @@ def get_site_settings(site):
|
||||
# ============================ 任务历史 ============================
|
||||
|
||||
|
||||
def create_task(site, kind):
|
||||
"""新建一条 pending 任务,返回其 id。"""
|
||||
def create_task(site, kind, trigger="manual", target_date="", force=False):
|
||||
"""新建一条 pending 任务(手动触发),返回其 id。trigger='manual'/'auto';
|
||||
target_date 为该任务的目标下载日期(YYYY-MM-DD,可为 '');force 是否强制重下。"""
|
||||
now = _now()
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO task_history (site, kind, status, started_at, finished_at, error) "
|
||||
"VALUES (?, ?, ?, ?, '', '')",
|
||||
(site, kind, TASK_PENDING, now),
|
||||
"INSERT INTO task_history "
|
||||
"(site, kind, status, started_at, finished_at, error, trigger, target_date, force) "
|
||||
"VALUES (?, ?, ?, ?, '', '', ?, ?, ?)",
|
||||
(site, kind, TASK_PENDING, now, trigger, target_date, 1 if force else 0),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
def create_task_if_idle(site, kind, trigger="auto", target_date=""):
|
||||
"""周期调度专用:若该 (site,kind) 已有 pending/running 任务则返回 None(跳过本次周期),
|
||||
否则建一条 pending 任务返回其 id。单连接内 check-then-insert,靠 SQLite 写锁把竞态压到忽略不计。
|
||||
|
||||
与 create_task 的区别:手动触发(POST /tasks)用 create_task(用户点的必建);周期 job 用本函数
|
||||
——上一次还没跑完时跳过,避免同 (site,kind) 任务堆积。手动建的任务会让紧随其后的周期 fire
|
||||
判到 inflight 而跳过,天然互斥。trigger='auto';target_date 为目标下载日期(YYYY-MM-DD)。"""
|
||||
now = _now()
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM task_history WHERE site=? AND kind=? "
|
||||
"AND status IN (?, ?) LIMIT 1",
|
||||
(site, kind, TASK_PENDING, TASK_RUNNING),
|
||||
).fetchone()
|
||||
if row:
|
||||
return None
|
||||
cur = conn.execute(
|
||||
"INSERT INTO task_history "
|
||||
"(site, kind, status, started_at, finished_at, error, trigger, target_date, force) "
|
||||
"VALUES (?, ?, ?, ?, '', '', ?, ?, 0)",
|
||||
(site, kind, TASK_PENDING, now, trigger, target_date),
|
||||
)
|
||||
conn.commit()
|
||||
return cur.lastrowid
|
||||
@@ -415,7 +617,7 @@ def create_task(site, kind):
|
||||
def update_task(task_id, status, error=None):
|
||||
"""更新任务状态。终态(success/no_data/failed)写入 finished_at。"""
|
||||
finished = _now() if status in (TASK_SUCCESS, TASK_NO_DATA, TASK_FAILED) else ""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
if finished:
|
||||
conn.execute(
|
||||
"UPDATE task_history SET status=?, finished_at=?, error=? WHERE id=?",
|
||||
@@ -431,9 +633,10 @@ def update_task(task_id, status, error=None):
|
||||
|
||||
def get_task(task_id):
|
||||
"""返回单条任务 dict,不存在返回 None。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, site, kind, status, started_at, finished_at, error "
|
||||
"SELECT id, site, kind, status, started_at, finished_at, error, "
|
||||
"trigger, target_date, force "
|
||||
"FROM task_history WHERE id=?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
@@ -447,14 +650,18 @@ def get_task(task_id):
|
||||
"started_at": row[4],
|
||||
"finished_at": row[5],
|
||||
"error": row[6],
|
||||
"trigger": row[7],
|
||||
"target_date": row[8],
|
||||
"force": bool(row[9]),
|
||||
}
|
||||
|
||||
|
||||
def list_tasks(limit=20):
|
||||
"""返回最近 limit 条任务(按 id 倒序)。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, site, kind, status, started_at, finished_at, error "
|
||||
"SELECT id, site, kind, status, started_at, finished_at, error, "
|
||||
"trigger, target_date, force "
|
||||
"FROM task_history ORDER BY id DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
@@ -467,6 +674,9 @@ def list_tasks(limit=20):
|
||||
"started_at": r[4],
|
||||
"finished_at": r[5],
|
||||
"error": r[6],
|
||||
"trigger": r[7],
|
||||
"target_date": r[8],
|
||||
"force": bool(r[9]),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
@@ -475,7 +685,7 @@ def list_tasks(limit=20):
|
||||
def fail_stale_tasks(reason: str = "服务重启,上轮未完成任务,请手动重跑") -> int:
|
||||
"""worker 启动时调用:把遗留的 pending/running 任务标记为 failed,实现重启自愈。
|
||||
返回被清理的任务数量。"""
|
||||
with sqlite3.connect(STATE_DB_PATH) as conn:
|
||||
with sqlite3.connect(STATE_DB_PATH, timeout=3.0) as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE task_history SET status=?, finished_at=?, error=? "
|
||||
"WHERE status IN (?, ?)",
|
||||
|
||||
@@ -268,6 +268,18 @@ _SQL_UNDELIVERED = """
|
||||
ingested_at = now()
|
||||
"""
|
||||
|
||||
_SQL_BAISHI_DAILY_STATS = """
|
||||
INSERT INTO baishi_daily_stats
|
||||
(site, business_date, expected_pieces, arrived_pieces, undelivered_pieces, raw)
|
||||
VALUES (%s,%s,%s,%s,%s,%s)
|
||||
ON CONFLICT (site, business_date) DO UPDATE SET
|
||||
expected_pieces = COALESCE(EXCLUDED.expected_pieces, baishi_daily_stats.expected_pieces),
|
||||
arrived_pieces = COALESCE(EXCLUDED.arrived_pieces, baishi_daily_stats.arrived_pieces),
|
||||
undelivered_pieces = COALESCE(EXCLUDED.undelivered_pieces, baishi_daily_stats.undelivered_pieces),
|
||||
raw = EXCLUDED.raw,
|
||||
ingested_at = now()
|
||||
"""
|
||||
|
||||
|
||||
# ============================== 入库 ==============================
|
||||
|
||||
@@ -314,9 +326,10 @@ def _ingest_actual(cur, site):
|
||||
cm = ACTUAL_COLMAP[site]
|
||||
df = pd.read_excel(path, dtype=str).fillna("")
|
||||
if site == "韵达":
|
||||
# 韵达业务清洗:抛弃「交接单号」为空的行(派件/签收等其他扫描无交接单号),
|
||||
# 韵达业务清洗:保留「交接单号」为空的行(到/接件扫描),
|
||||
# 抛弃「交接单号」不为空的行(派件/签收等,属重复数据)。
|
||||
# 再按子单号去重(一件多扫只留一条;清洗后子单号已天然唯一,drop 为保险)。
|
||||
df = df[df["交接单号"].astype(str).str.strip() != ""]
|
||||
df = df[df["交接单号"].astype(str).str.strip() == ""]
|
||||
df = df.drop_duplicates(subset=[cm["piece"]], keep="last")
|
||||
rows = []
|
||||
for r in df.to_dict("records"):
|
||||
@@ -368,6 +381,34 @@ def _ingest_undelivered_baishi(cur):
|
||||
return len(rows)
|
||||
|
||||
|
||||
def upsert_baishi_daily_stats(exp, arr, business_date=None):
|
||||
"""直接落库百世当日应到/实到基数(应扫/已扫,站级日聚合)。
|
||||
供 baishi 下载时抓到基数后直接调用(一步落库,不绕 state_store→store)。
|
||||
business_date 默认今天(百世固定当天)。best-effort:失败只告警,不影响下载流程。"""
|
||||
biz = business_date or date.today()
|
||||
if exp is None and arr is None:
|
||||
return
|
||||
undel = (exp - arr) if (exp is not None and arr is not None) else None
|
||||
try:
|
||||
with _connect(_load_pg_config()["dbname"]) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
_SQL_BAISHI_DAILY_STATS,
|
||||
(
|
||||
"百世",
|
||||
biz,
|
||||
exp,
|
||||
arr,
|
||||
undel,
|
||||
Jsonb({"expected": exp, "arrived": arr, "undelivered": undel}),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
print(f" [基数] 百世 {biz}: 应扫 {exp} / 已扫 {arr} / 未扫 {undel}")
|
||||
except Exception as e:
|
||||
print(f" [基数] 百世 {biz} 入库失败(不影响下载): {e}")
|
||||
|
||||
|
||||
def ingest(site=None):
|
||||
"""入库:指定 site 则单站(百世只入未到),否则全站。返回总条数。"""
|
||||
dates = _read_business_dates()
|
||||
@@ -422,6 +463,71 @@ def ingest_task(site, kind):
|
||||
return total
|
||||
|
||||
|
||||
def get_existing_handover_nos(site):
|
||||
"""查该站点已落库的交接单号集合(expected_record.handover_no)。
|
||||
供"提交导出任务前"去重:已落库的交接单号不再重复提交导出任务。
|
||||
PG 不可用(cpolar 抖动等)时返回空集 + 告警,调用方按"未确认存在"处理
|
||||
(继续提交导出,UPSERT 兜底,绝不因去重查询失败而漏数据)。"""
|
||||
try:
|
||||
with _connect(_load_pg_config()["dbname"]) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT handover_no FROM expected_record "
|
||||
"WHERE site=%s AND handover_no IS NOT NULL AND handover_no <> ''",
|
||||
(site,),
|
||||
)
|
||||
return {str(r[0]).strip() for r in cur.fetchall()}
|
||||
except Exception as e:
|
||||
print(f">> [去重] 查询已落库交接单号失败({site}),本次不去重: {e}")
|
||||
return set()
|
||||
|
||||
|
||||
# ============================== PG 数据存在性查询 ==============================
|
||||
|
||||
|
||||
def has_data(site, kind, target_date):
|
||||
"""查询 PG:指定站点在 target_date 是否有业务数据。
|
||||
target_date: str 'YYYY-MM-DD' 或 date 对象。
|
||||
返回 (has_rows: bool, count: int)。
|
||||
PG 不可达时返回 (False, 0),不抛异常——调用方按「未确认存在」处理。
|
||||
|
||||
kind 路由:
|
||||
expected → expected_record (business_date)
|
||||
actual → actual_record (scan_time::date)
|
||||
undelivered → 百世: baishi_daily_stats;4 站: 不单独查(由调用方 expected∧actual 派生)
|
||||
"""
|
||||
if site == "百世" and kind == "undelivered":
|
||||
sql = (
|
||||
"SELECT COUNT(*) FROM baishi_daily_stats"
|
||||
" WHERE site = %s AND business_date = %s"
|
||||
)
|
||||
params = (site, target_date)
|
||||
elif kind == "expected":
|
||||
sql = (
|
||||
"SELECT COUNT(*) FROM expected_record"
|
||||
" WHERE site = %s AND business_date = %s"
|
||||
)
|
||||
params = (site, target_date)
|
||||
elif kind == "actual":
|
||||
sql = (
|
||||
"SELECT COUNT(*) FROM actual_record"
|
||||
" WHERE site = %s AND scan_time::date = %s"
|
||||
)
|
||||
params = (site, target_date)
|
||||
else:
|
||||
return (False, 0)
|
||||
try:
|
||||
with _connect(_load_pg_config()["dbname"]) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
row = cur.fetchone()
|
||||
cnt = int(row[0]) if row else 0
|
||||
return (cnt > 0, cnt)
|
||||
except Exception as e:
|
||||
print(f">> [状态] PG 查询 {site}/{kind}/{target_date} 失败: {e}")
|
||||
return (False, 0)
|
||||
|
||||
|
||||
# ============================== 命令行 ==============================
|
||||
|
||||
|
||||
|
||||
14
schema.sql
14
schema.sql
@@ -24,6 +24,7 @@ CREATE TABLE IF NOT EXISTS expected_record (
|
||||
UNIQUE (site, waybill_no)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_expected_site_date ON expected_record (site, business_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_expected_handover ON expected_record (site, handover_no);
|
||||
|
||||
-- 实到货物(扫描件级:一扫描一行;每扫描一件系统生成一个单号)
|
||||
CREATE TABLE IF NOT EXISTS actual_record (
|
||||
@@ -52,3 +53,16 @@ CREATE TABLE IF NOT EXISTS undelivered_record (
|
||||
ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (site, piece_no)
|
||||
);
|
||||
|
||||
-- 百世日聚合(应扫/已扫基数:站级日聚合,区别于运单级/件级/子单级表)
|
||||
CREATE TABLE IF NOT EXISTS baishi_daily_stats (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
site TEXT NOT NULL, -- 百世
|
||||
business_date DATE NOT NULL, -- 业务日期(百世固定当天)
|
||||
expected_pieces INTEGER, -- 应扫(应到基数)
|
||||
arrived_pieces INTEGER, -- 已扫(实到基数)
|
||||
undelivered_pieces INTEGER, -- 未扫(=应扫-已扫,任一缺失则 NULL)
|
||||
raw JSONB NOT NULL, -- 原始抓取值
|
||||
ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (site, business_date)
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user