9d3d3cbb78
安能站点第一阶段：把应到数据（运单信息）下载流程接入主路由。 - site_anneng.py：用页面级 CDP 驱动「安能全网门户」Electron 应用（Playwright connect_over_cdp 被 Electron 浏览器域 CDP 拦截，故走页面级）。完整应到流程： 菜单导航 → 进站交接单查询 → 设日期查询（“带小数点的 0”为加载完成判据） → 逐条交接单双击进运单信息并核对单号 → 导出（选中字段+→箭头转移全部列） → 关闭查询 tab → 导出下载页轮询 → 免对话框下载（x-auth+TGC 直连 GET） → 合并为 安能-应到货物数据.xlsx。CDP 端口可配；anneng_ready() 供路由就绪轮询。 - main_router.py：以调试模式启动安能（动态空闲端口，避免端口冲突），经 CDP 判断首页就绪，菜单加 [10] 应到下载，退出时关闭应用；网页站点逻辑不变。 - config.example.yaml：补充 anneng.query_days 与 anneng.app_path 说明。 - docs/安能门户CDP连接指南.md：连接方案、独立 webContents/tab 处理、免对话框下载。 - .gitignore：忽略 Archive/（早期探查脚本归档）。 Co-Authored-By: Claude <noreply@anthropic.com>
524 lines
22 KiB
Python
524 lines
22 KiB
Python
# main_router.py
|
||
|
||
import os
|
||
import socket
|
||
import subprocess
|
||
import time
|
||
import urllib.request
|
||
import yaml
|
||
import pandas as pd
|
||
from playwright.sync_api import sync_playwright
|
||
|
||
from paths import DOWNLOAD_DIR, CONFIG_PATH
|
||
|
||
# 导入抽离出去的各个网点模块
|
||
import site_shunxin
|
||
import site_baishi
|
||
import site_zto
|
||
import site_yunda
|
||
import site_anneng
|
||
|
||
# 定义网点及对应的初始登录 URL
|
||
SITES_CONFIG = {
|
||
"顺心": "https://sxne.sxjdfreight.com",
|
||
"百世": "https://v5.800best.com",
|
||
"中通": "https://ws.zto56.com/",
|
||
"韵达": "https://ky-sso.yunda56.com",
|
||
}
|
||
|
||
# ====================================================================
|
||
# 站点就绪特征：每个站点登录成功进入工作台后的标志性控件
|
||
# ====================================================================
|
||
READY_SELECTORS = {
|
||
"顺心": 'h1:has-text("盟商门户网")',
|
||
"百世": 'h1[title="百世快运"]',
|
||
"中通": '.logo:has-text("网点版")',
|
||
"韵达": '.el-menu-item:has-text("首页")',
|
||
}
|
||
|
||
# 安能是 Electron 桌面应用（不是 Playwright 打开的网页），需要单独启动。
|
||
APP_SITES = {"安能"}
|
||
|
||
|
||
def _find_free_port():
|
||
"""让操作系统分配一个空闲端口，避免固定端口冲突。"""
|
||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||
s.bind(("127.0.0.1", 0))
|
||
return s.getsockname()[1]
|
||
|
||
|
||
def _wait_cdp_up(port, timeout=60.0):
|
||
"""轮询直到 CDP 调试端口就绪（应用启动需要时间）。"""
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
with urllib.request.urlopen(
|
||
f"http://localhost:{port}/json/version", timeout=2
|
||
) as resp:
|
||
if resp.status == 200:
|
||
return True
|
||
except Exception:
|
||
pass
|
||
time.sleep(1)
|
||
return False
|
||
|
||
|
||
def launch_anneng(app_path):
|
||
"""以调试模式启动安能 Electron 应用（自动选取空闲端口），返回子进程对象。
|
||
|
||
启动后请在应用内手动登录；就绪状态由 run_multi_site_daemon 的就绪轮询判断。
|
||
"""
|
||
port = _find_free_port()
|
||
print(f">> 以调试模式启动【安能】应用（端口 {port}）：{app_path}")
|
||
proc = subprocess.Popen([app_path, f"--remote-debugging-port={port}"])
|
||
site_anneng.set_cdp_port(port)
|
||
if not _wait_cdp_up(port):
|
||
raise RuntimeError(
|
||
f"安能应用调试端口 {port} 未就绪——可能应用已在运行（单实例），"
|
||
"请先关闭已有的安能窗口再试"
|
||
)
|
||
return proc
|
||
|
||
|
||
def task_process_undelivered_data(site_name="顺心"):
|
||
"""应到未到比对：找出应到但未实到的运单 (按站点前缀)"""
|
||
print(f"\n▶ 开始执行【{site_name} - 应到未到数据处理】任务...")
|
||
|
||
download_dir = DOWNLOAD_DIR
|
||
expected_path = os.path.join(download_dir, f"{site_name}-应到货物数据.xlsx")
|
||
actual_path = os.path.join(download_dir, f"{site_name}-实到货物数据.xlsx")
|
||
output_path = os.path.join(download_dir, f"{site_name}-应到未到货物数据.xlsx")
|
||
|
||
if not os.path.exists(expected_path):
|
||
print(f"❌ 错误：找不到【{site_name}-应到货物数据】主文档：{expected_path}")
|
||
return
|
||
|
||
if not os.path.exists(actual_path):
|
||
print(f"❌ 错误：找不到【{site_name}-实到货物数据】主文档：{actual_path}")
|
||
return
|
||
|
||
try:
|
||
print(">> 正在载入本地 Excel 文档...")
|
||
# 全部按字符串读取并保留空串：避免 18 位运单号被当成浮点数丢精度，也避免空单元格变成 NaN。
|
||
df_expected = pd.read_excel(expected_path, dtype=str, keep_default_na=False)
|
||
df_actual = pd.read_excel(actual_path, dtype=str, keep_default_na=False)
|
||
|
||
if "运单号" not in df_expected.columns or "运单号" not in df_actual.columns:
|
||
print("❌ 校验失败：数据源缺少【运单号】字段，请检查导出配置。")
|
||
return
|
||
|
||
print(">> 正在比对应到与实到数据...")
|
||
|
||
# 统一运单号为去空白字符串，消除 int/float 与 str 混读导致 isin 永不命中的隐患
|
||
df_expected["运单号"] = df_expected["运单号"].astype(str).str.strip()
|
||
df_actual["运单号"] = df_actual["运单号"].astype(str).str.strip()
|
||
|
||
# 剔除空白运单号，避免空值被误判为“应到未到”
|
||
df_expected = df_expected[df_expected["运单号"] != ""]
|
||
actual_set = set(df_actual["运单号"]) - {""}
|
||
|
||
# Left Anti-Join：在应到中找出不存在于实到里的运单号
|
||
df_undelivered = df_expected[~df_expected["运单号"].isin(actual_set)]
|
||
|
||
target_columns = ["班次号", "交接单号", "运单号"]
|
||
available_columns = [
|
||
col for col in target_columns if col in df_undelivered.columns
|
||
]
|
||
df_output = df_undelivered[available_columns]
|
||
|
||
print(f">> 比对完成，共筛选出【应到未到】运单: {len(df_output)} 条。")
|
||
|
||
df_output.to_excel(output_path, index=False)
|
||
print(f"====================================================")
|
||
print(f" 比对完成，结果已输出。")
|
||
print(f" 📁 输出路径: {output_path}")
|
||
print(f"====================================================")
|
||
|
||
except Exception as e:
|
||
print(f"❌ 比对过程中发生异常: {e}")
|
||
|
||
|
||
# ====================================================================
|
||
# 自动化测试入口
|
||
# ====================================================================
|
||
# 每个(非百世)站点的交叉测试序列：覆盖两种下载流程之间的全部 4 种相邻转换，
|
||
# 用于验证无论上一个流程把页面留在什么状态，下一个流程都能正常运行：
|
||
# 应到->实到、实到->应到、应到->应到、实到->实到
|
||
CROSS_TEST_SEQUENCE = ["expected", "actual", "expected", "expected", "actual", "actual"]
|
||
|
||
|
||
def run_automation_test(pages_map):
|
||
"""自动化测试入口：按交叉序列逐个跑通各站点的下载流程，结束后打印统计报告。
|
||
|
||
判定规则：流程函数返回 False 或抛出异常记为 FAIL，其余记为 PASS。
|
||
"""
|
||
# 站点 -> {流程键: (中文名, 流程函数)}；百世为单流程，单独处理
|
||
flow_table = {
|
||
"顺心": {
|
||
"expected": ("应到", site_shunxin.shunxin_expected_download),
|
||
"actual": ("实到", site_shunxin.shunxin_actual_download),
|
||
},
|
||
"中通": {
|
||
"expected": ("应到", site_zto.zto_expected_download),
|
||
"actual": ("实到", site_zto.zto_actual_download),
|
||
},
|
||
"韵达": {
|
||
"expected": ("应到", site_yunda.yunda_expected_download),
|
||
"actual": ("实到", site_yunda.yunda_actual_download),
|
||
},
|
||
}
|
||
|
||
# 构建测试计划：[(站点, 流程中文名, 流程函数), ...]
|
||
plan = []
|
||
for site_name, flows in flow_table.items():
|
||
if site_name not in pages_map:
|
||
continue
|
||
for flow_key in CROSS_TEST_SEQUENCE:
|
||
label, func = flows[flow_key]
|
||
plan.append((site_name, label, func))
|
||
# 百世：单流程，跑一次即可
|
||
if "百世" in pages_map:
|
||
plan.append(("百世", "应到未到", site_baishi.baishi_download_undelivered_data))
|
||
|
||
if not plan:
|
||
print("\n⚠️ 当前没有已就绪的站点，无法执行自动化测试。")
|
||
return
|
||
|
||
total = len(plan)
|
||
print("\n====================================================")
|
||
print(f"自动化测试开始，共 {total} 个步骤。")
|
||
print("（双流程站点按应到/实到交叉序列执行，覆盖全部相邻转换）")
|
||
print("====================================================")
|
||
|
||
results = [] # [(站点, 流程, 状态, 耗时秒, 错误信息)]
|
||
for idx, (site_name, label, func) in enumerate(plan, start=1):
|
||
print("\n----------------------------------------------------")
|
||
print(f"[步骤 {idx}/{total}] 站点【{site_name}】流程【{label}】")
|
||
print("----------------------------------------------------")
|
||
page = pages_map[site_name]
|
||
start = time.time()
|
||
status = "PASS"
|
||
err = ""
|
||
try:
|
||
page.bring_to_front()
|
||
ret = func(page)
|
||
if ret is False:
|
||
status = "FAIL"
|
||
err = "流程返回失败状态"
|
||
except Exception as e:
|
||
status = "FAIL"
|
||
err = str(e)
|
||
elapsed = time.time() - start
|
||
results.append((site_name, label, status, elapsed, err))
|
||
print(f">> 步骤结果: {status} (耗时 {elapsed:.1f}s)")
|
||
|
||
_print_test_report(results)
|
||
|
||
|
||
def _print_test_report(results):
|
||
"""打印自动化测试统计报告。"""
|
||
passed = sum(1 for r in results if r[2] == "PASS")
|
||
failed = len(results) - passed
|
||
|
||
print("\n====================================================")
|
||
print("自动化测试统计报告")
|
||
print("====================================================")
|
||
for i, (site_name, label, status, elapsed, err) in enumerate(results, start=1):
|
||
mark = "✅" if status == "PASS" else "❌"
|
||
print(
|
||
f" {i:>2}. {mark} {status} {site_name} - {label} (耗时 {elapsed:.1f}s)"
|
||
)
|
||
if err:
|
||
note = err if len(err) <= 60 else err[:57] + "..."
|
||
print(f" 说明: {note}")
|
||
print("----------------------------------------------------")
|
||
print(f" 合计 {len(results)} 步：通过 {passed}，失败 {failed}")
|
||
if failed == 0:
|
||
print(" ✅ 全部流程跑通。")
|
||
else:
|
||
print(" ❌ 存在失败流程，请结合上方说明与运行日志排查。")
|
||
print("====================================================")
|
||
|
||
|
||
def run_multi_site_daemon():
|
||
"""多站点自动化主控流程"""
|
||
|
||
# 1. 读取配置文件
|
||
debug_mode = False
|
||
debug_target = ""
|
||
anneng_app_path = ""
|
||
try:
|
||
if os.path.exists(CONFIG_PATH):
|
||
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
|
||
config = yaml.safe_load(f) or {}
|
||
debug_mode = (config.get("debug", {}) or {}).get("enabled", False)
|
||
debug_target = (config.get("debug", {}) or {}).get("target_site", "")
|
||
anneng_app_path = (config.get("anneng", {}) or {}).get("app_path", "")
|
||
except Exception as e:
|
||
print(f"⚠️ 读取 config.yaml 异常，将使用全量模式启动: {e}")
|
||
|
||
# 动态确定需要挂载启动的网页站点；安能（Electron 应用）单独标记
|
||
anneng_active = False
|
||
if debug_mode:
|
||
if debug_target in SITES_CONFIG:
|
||
print(f"\n🛠️ 【调试模式】仅加载目标站点: [{debug_target}]")
|
||
active_sites = {debug_target: SITES_CONFIG[debug_target]}
|
||
elif debug_target == "安能":
|
||
print(f"\n🛠️ 【调试模式】仅加载目标站点: [安能]")
|
||
active_sites = {}
|
||
anneng_active = True
|
||
else:
|
||
active_sites = dict(SITES_CONFIG)
|
||
anneng_active = True
|
||
else:
|
||
active_sites = dict(SITES_CONFIG)
|
||
anneng_active = True
|
||
|
||
if anneng_active and not anneng_app_path:
|
||
print("⚠️ 已启用安能但 config.yaml 未配置 anneng.app_path，将跳过安能。")
|
||
anneng_active = False
|
||
|
||
with sync_playwright() as p:
|
||
browser = p.chromium.launch(headless=False)
|
||
context = browser.new_context(viewport={"width": 1920, "height": 1080})
|
||
|
||
pages_map = {}
|
||
|
||
print("\n====================================================")
|
||
print("【启动】正在打开各站点页面...")
|
||
print("====================================================")
|
||
|
||
for site_name, url in active_sites.items():
|
||
print(f">> 正在启动【{site_name}】页面: {url}")
|
||
page = context.new_page()
|
||
page.goto(url)
|
||
pages_map[site_name] = page
|
||
|
||
# 安能：以调试模式启动 Electron 应用（非 Playwright 网页）
|
||
anneng_proc = None
|
||
if anneng_active:
|
||
try:
|
||
anneng_proc = launch_anneng(anneng_app_path)
|
||
pages_map["安能"] = True # 哨兵：表示已启动（无 Playwright page 对象）
|
||
except Exception as e:
|
||
print(f"⚠️ 启动安能应用失败，已跳过安能：{e}")
|
||
anneng_active = False
|
||
|
||
print("\n====================================================")
|
||
print("【登录检测】正在准备各站点登录...")
|
||
print("====================================================")
|
||
|
||
# 针对支持纯代码自动登录的站点，在此处前置注入登录事件
|
||
if "韵达" in pages_map:
|
||
try:
|
||
pages_map["韵达"].bring_to_front()
|
||
site_yunda.yunda_login(pages_map["韵达"])
|
||
except Exception as e:
|
||
print(f" ⚠️ 韵达前置自动登录模块发生波动: {e}")
|
||
|
||
# ====================================================================
|
||
# 就绪轮询 (Ready Guard Polling)
|
||
# 自动识别各站点登录完成状态，无需手动回车
|
||
# ====================================================================
|
||
ready_status = {site: False for site in active_sites.keys()}
|
||
if anneng_active:
|
||
ready_status["安能"] = False
|
||
|
||
print("\n>> 正在轮询各站点就绪状态 (自动登录或手动登录均可)...")
|
||
while not all(ready_status.values()):
|
||
for site_name in list(ready_status.keys()):
|
||
if ready_status[site_name]:
|
||
continue
|
||
try:
|
||
if site_name == "安能":
|
||
# 安能走 CDP 判断主页是否就绪（连不上返回 False，不抛异常）
|
||
ok = site_anneng.anneng_ready()
|
||
else:
|
||
# 0.5 秒轻量探测，避免阻塞主循环
|
||
ok = (
|
||
pages_map[site_name]
|
||
.locator(READY_SELECTORS[site_name])
|
||
.is_visible(timeout=500)
|
||
)
|
||
if ok:
|
||
ready_status[site_name] = True
|
||
print(f" ✅ 【{site_name}】已检测到主页，登录就绪。")
|
||
except Exception:
|
||
pass
|
||
|
||
pending_sites = [s for s, ready in ready_status.items() if not ready]
|
||
if pending_sites:
|
||
print(
|
||
f" ⏳ 等待以下站点完成登录: [{', '.join(pending_sites)}] ... "
|
||
"(请在浏览器/应用中操作)"
|
||
)
|
||
time.sleep(3) # 等待 3 秒后进行下一轮检查
|
||
|
||
print("\n====================================================")
|
||
print("【准备】所有站点已就绪，正在清理初始弹窗...")
|
||
print("====================================================")
|
||
|
||
# 顺心：处理初始弹窗
|
||
if "顺心" in pages_map:
|
||
try:
|
||
sx_page = pages_map["顺心"]
|
||
sx_page.bring_to_front()
|
||
print(">> 正在处理【顺心】弹窗与遮罩...")
|
||
sx_page.locator("a").nth(4).click(timeout=2000)
|
||
sx_page.wait_for_timeout(500)
|
||
sx_page.get_by_role("button", name="Close").click(timeout=2000)
|
||
sx_page.wait_for_timeout(500)
|
||
sx_page.get_by_role("button", name="不再询问").click(timeout=2000)
|
||
print(" ✅ 【顺心】初始弹窗处理完成。")
|
||
except Exception:
|
||
pass # 环境可能很干净没有弹窗，无视报错
|
||
|
||
# 百世：循环关闭初始弹窗
|
||
if "百世" in pages_map:
|
||
try:
|
||
bs_page = pages_map["百世"]
|
||
bs_page.bring_to_front()
|
||
print(">> 正在处理【百世】阅读完毕与关闭按钮...")
|
||
for round_idx in range(4):
|
||
handled_any = False
|
||
try:
|
||
read_btns = bs_page.locator("button:has-text('阅读完毕')")
|
||
if read_btns.count() > 0:
|
||
for i in range(read_btns.count()):
|
||
if read_btns.nth(i).is_visible(timeout=500):
|
||
read_btns.nth(i).click()
|
||
handled_any = True
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if bs_page.locator("button:has-text('关 闭')").is_visible(
|
||
timeout=500
|
||
):
|
||
bs_page.locator("button:has-text('关 闭')").click()
|
||
handled_any = True
|
||
except Exception:
|
||
pass
|
||
if not handled_any:
|
||
break
|
||
bs_page.wait_for_timeout(800)
|
||
print(" ✅ 【百世】初始弹窗处理完成。")
|
||
except Exception:
|
||
pass
|
||
|
||
if "中通" in pages_map:
|
||
print(" ✅ 【中通】已就绪。")
|
||
|
||
if "韵达" in pages_map:
|
||
print(" ✅ 【韵达】已就绪。")
|
||
|
||
if "安能" in pages_map:
|
||
print(" ✅ 【安能】已就绪。")
|
||
|
||
def is_site_ready(site_name):
|
||
if site_name not in pages_map:
|
||
print(f"\n🚫 站点 [{site_name}] 未加载（当前为调试模式），已跳过。")
|
||
return False
|
||
return True
|
||
|
||
while True:
|
||
print("\n====================================================")
|
||
print(" 物流数据下载主菜单 ")
|
||
if debug_mode:
|
||
print(f" [ 调试模式，仅加载: {debug_target} ]")
|
||
print("====================================================")
|
||
print(" 模块一：【顺心】数据处理流")
|
||
print(" [1] 执行 - 应到货物数据下载")
|
||
print(" [2] 执行 - 实到货物数据下载")
|
||
print("-" * 52)
|
||
print(" 模块二：【百世】数据处理流")
|
||
print(" [3] 执行 - 一键提取应到未到异常数据")
|
||
print("-" * 52)
|
||
print(" 模块三：【中通】数据处理流")
|
||
print(" [4] 执行 - 应到货物数据下载")
|
||
print(" [5] 执行 - 实到货物数据下载")
|
||
print("-" * 52)
|
||
print(" 模块四：【韵达】数据处理流")
|
||
print(" [6] 执行 - 应到货物数据下载")
|
||
print(" [7] 执行 - 实到货物数据下载")
|
||
print("-" * 52)
|
||
print(" 模块五：【安能】数据处理流（Electron 应用）")
|
||
print(" [10] 执行 - 应到货物数据下载（运单信息）")
|
||
print("-" * 52)
|
||
print(" 自动化测试")
|
||
print(" [8] 执行 - 全站点下载流程自动化测试 (交叉跑通校验)")
|
||
print("-" * 52)
|
||
print(" 全局离线数据处理")
|
||
print(" [9] 执行 - 异常数据清洗比对 (Left Anti-Join)")
|
||
print("-" * 52)
|
||
print(" [0] 退出系统")
|
||
print("====================================================")
|
||
|
||
choice = input("请输入任务编号并回车: ")
|
||
|
||
try:
|
||
if choice == "1" and is_site_ready("顺心"):
|
||
pages_map["顺心"].bring_to_front()
|
||
site_shunxin.shunxin_expected_download(pages_map["顺心"])
|
||
elif choice == "2" and is_site_ready("顺心"):
|
||
pages_map["顺心"].bring_to_front()
|
||
site_shunxin.shunxin_actual_download(pages_map["顺心"])
|
||
elif choice == "3" and is_site_ready("百世"):
|
||
pages_map["百世"].bring_to_front()
|
||
site_baishi.baishi_download_undelivered_data(pages_map["百世"])
|
||
elif choice == "4" and is_site_ready("中通"):
|
||
pages_map["中通"].bring_to_front()
|
||
site_zto.zto_expected_download(pages_map["中通"])
|
||
elif choice == "5" and is_site_ready("中通"):
|
||
pages_map["中通"].bring_to_front()
|
||
site_zto.zto_actual_download(pages_map["中通"])
|
||
elif choice == "6" and is_site_ready("韵达"):
|
||
pages_map["韵达"].bring_to_front()
|
||
site_yunda.yunda_expected_download(pages_map["韵达"])
|
||
elif choice == "7" and is_site_ready("韵达"):
|
||
pages_map["韵达"].bring_to_front()
|
||
site_yunda.yunda_actual_download(pages_map["韵达"])
|
||
elif choice == "10" and is_site_ready("安能"):
|
||
site_anneng.anneng_expected_download()
|
||
elif choice == "8":
|
||
run_automation_test(pages_map)
|
||
elif choice == "9":
|
||
site_name = input(
|
||
"请输入要比对的网点名称 (如 顺心/中通/韵达): "
|
||
).strip()
|
||
if not site_name:
|
||
site_name = "顺心"
|
||
task_process_undelivered_data(site_name)
|
||
elif choice == "0":
|
||
print("\n正在关闭浏览器并退出...")
|
||
break
|
||
else:
|
||
if choice not in [
|
||
"1",
|
||
"2",
|
||
"3",
|
||
"4",
|
||
"5",
|
||
"6",
|
||
"7",
|
||
"8",
|
||
"9",
|
||
"10",
|
||
"0",
|
||
]:
|
||
print("\n⚠️ 无效输入，请查证后回车。")
|
||
except Exception as e:
|
||
print(f"❌ 任务调度异常: {e}")
|
||
|
||
browser.close()
|
||
if anneng_proc is not None:
|
||
try:
|
||
anneng_proc.terminate()
|
||
print("已关闭安能应用。")
|
||
except Exception:
|
||
pass
|
||
print("程序已退出。")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run_multi_site_daemon()
|