feat: dedup expected data by handover_no before export submit

- store.get_existing_handover_nos: query PG expected_record.handover_no

- inject dedup skip before submitting export in zto/yunda/anneng/shunxin

- shunxin reads RTS handover_no from waybill-list view (method 1)

- force-redownload switch threaded via task_spec -> dispatch -> impl

- schema: add idx_expected_handover index

Co-Authored-By: Claude <noreply@anthropic.com>

inbound_verify/cli/server.py

@@ -187,6 +187,9 @@ app = FastAPI(title="InboundVerify 服务端", lifespan=lifespan)
 class TaskRequest(BaseModel):
     site: str
     kind: str
+    force: bool = (
+        False  # 强制重下：忽略已落库去重，重新提交所有班次/交接单的导出任务（默认关）
+    )
 
 
 @app.post("/tasks")
@@ -200,7 +203,7 @@ def create_task(req: TaskRequest):
     if (req.site, req.kind) not in TASK_HANDLERS:
         raise HTTPException(status_code=400, detail=f"无效任务: {req.site}/{req.kind}")
     task_id = state_store.create_task(req.site, req.kind)
-    task_queue.put((task_id, {"site": req.site, "kind": req.kind}))
+    task_queue.put((task_id, {"site": req.site, "kind": req.kind, "force": req.force}))
     return {"task_id": task_id}
 
 

inbound_verify/runtime.py

@@ -454,14 +454,14 @@ def _web_handler(site, download_func):
     焦点；交互模式置顶便于调试。顺心是 page 列表，置顶标志透传给 shunxin_download。
     """
 
-    def handler(ctx):
+    def handler(ctx, force=False):
         pg = ctx.pages_map[site]
         if isinstance(pg, list):
             # 顺心双账号：置顶与否交给 shunxin_download 在逐账号循环里按 foreground 决定
-            return download_func(pg, foreground=ctx.foreground)
+            return download_func(pg, foreground=ctx.foreground, force=force)
         if ctx.foreground:
             pg.bring_to_front()
-        return download_func(pg)
+        return download_func(pg, force=force)
 
     return handler
 
@@ -470,11 +470,13 @@ def _site_undelivered_handler(site):
     """4 站未到：下应到+实到 → 比对写 downloads/<站>-未到数据.xlsx。
     任一下载失败 → 清掉旧未到文件、返回 False（前端不展示陈旧未到）。"""
 
-    def handler(ctx):
+    def handler(ctx, force=False):
         # 各站下载入口约定返回 True/False；顺心历史返回 None（视为成功，与 dispatch 一致）
-        exp_ok = TASK_HANDLERS[(site, "expected")](ctx) is not False
+        exp_ok = TASK_HANDLERS[(site, "expected")](ctx, force) is not False
         act_ok = (
-            (TASK_HANDLERS[(site, "actual")](ctx) is not False) if exp_ok else False
+            (TASK_HANDLERS[(site, "actual")](ctx, force) is not False)
+            if exp_ok
+            else False
         )
         if exp_ok and act_ok:
             return compare.write_site_file(site)
@@ -502,10 +504,14 @@ TASK_HANDLERS = {
     ("韵达", "expected"): _web_handler("韵达", yunda.yunda_expected_download),
     ("韵达", "actual"): _web_handler("韵达", yunda.yunda_actual_download),
     ("韵达", "undelivered"): _site_undelivered_handler("韵达"),
-    ("安能", "expected"): lambda ctx: anneng.anneng_expected_download(),
-    ("安能", "actual"): lambda ctx: anneng.anneng_actual_download(),
+    ("安能", "expected"): lambda ctx, force=False: anneng.anneng_expected_download(
+        force=force
+    ),
+    ("安能", "actual"): lambda ctx, force=False: anneng.anneng_actual_download(
+        force=force
+    ),
     ("安能", "undelivered"): _site_undelivered_handler("安能"),
-    ("__compare__", "compare"): lambda ctx: (compare.main() or True),
+    ("__compare__", "compare"): lambda ctx, force=False: (compare.main() or True),
 }
 
 
@@ -594,7 +600,7 @@ def dispatch_task(ctx, task_spec):
     if handler is None:
         return (state_store.TASK_FAILED, f"未知任务: {site}/{kind}")
     try:
-        ret = handler(ctx)
+        ret = handler(ctx, bool(task_spec.get("force", False)))
         if ret is False:
             return (state_store.TASK_FAILED, "任务执行失败（重试耗尽）")
         _record_business_date(site, kind)

inbound_verify/sites/anneng.py

@@ -855,13 +855,15 @@ def _load_query_days():
     return max(1, days)
 
 
-def anneng_expected_download():
+def anneng_expected_download(force=False):
     """安能：应到货物数据下载（内部含异常兜底重试，路由层无感）。"""
 
-    return with_retry("安能", "应到", anneng_expected_download_impl, anneng_reset)
+    return with_retry(
+        "安能", "应到", lambda: anneng_expected_download_impl(force=force), anneng_reset
+    )
 
 
-def anneng_expected_download_impl():
+def anneng_expected_download_impl(force=False):
     """安能：应到货物数据（运单信息）下载，完整流程（单次执行，无重试；供自动化测试用）。"""
     print("\n▶ 开始执行【安能 - 应到货物数据下载】任务 ...")
     download_dir = DOWNLOAD_DIR
@@ -880,6 +882,20 @@ def anneng_expected_download_impl():
 
     main_cdp = find_main_page_cdp()
     export_times = []
+
+    # 【去重】加载本站已落库交接单号；force=True 或查询失败时 existing=空集（不去重）
+    if force:
+        existing = set()
+        print(">> [去重] 强制重下，跳过去重。")
+    else:
+        try:
+            from inbound_verify import store
+
+            existing = store.get_existing_handover_nos("安能")
+        except Exception as _e:
+            existing = set()
+            print(f">> [去重] 加载已落库交接单号失败，本次不去重: {_e}")
+
     try:
         # 1) 确认主页就绪并导航到“进站交接单查询”
         wait_home_ready(main_cdp)
@@ -918,6 +934,10 @@ def anneng_expected_download_impl():
 
             for i, ewbs_no in enumerate(target_ids, start=1):
                 print(f"   ⏳ [{i}/{len(target_ids)}] 交接单号 {ewbs_no}")
+                # 【去重】已落库则跳过：不双击、不导出、不 append export_times
+                if ewbs_no in existing:
+                    print(f"      ⏭️ 交接单号 {ewbs_no} 已落库，跳过。")
+                    continue
                 activate_tab(tab_cdp, "交接单信息")
                 time.sleep(0.3)
                 if not dblclick_jiaojie_dan_row(tab_cdp, ewbs_no):
@@ -936,6 +956,11 @@ def anneng_expected_download_impl():
         close_tab_by_label(main_cdp, "进站交接单查询")
         time.sleep(0.8)
 
+        # 【去重兜底】全部已落库/无数据 → 无导出任务，查询 tab 已关，跳过下载段
+        if not export_times:
+            print(">> 本次无新交接单需导出（全部已落库或无数据），结束。")
+            return True
+
         # 5) 打开导出下载 tab，轮询并下载
         print(">> 打开【导出下载】tab ...")
         export_cdp = ensure_tab_open(main_cdp, "导出下载", EXPORT_TAB_URL_HINT)
@@ -1233,7 +1258,7 @@ def _save_actual(rows, download_dir):
     print("====================================================")
 
 
-def anneng_actual_download():
+def anneng_actual_download(force=False):
     """安能：实到数据下载（内部含异常兜底重试，路由层无感）。"""
 
     return with_retry("安能", "实到", anneng_actual_download_impl, anneng_reset)

inbound_verify/sites/baishi.py

@@ -137,7 +137,7 @@ def _close_tab(page, tab_name):
         print(f"   ⚠️ 关闭标签页【{tab_name}】时出错: {e}")
 
 
-def baishi_download_undelivered_data(page):
+def baishi_download_undelivered_data(page, force=False):
     """百世：一键提取应到未到（当日未扫）数据（内部含异常兜底重试，路由层无感）。"""
 
     return with_retry(

inbound_verify/sites/shunxin.py

@@ -149,7 +149,7 @@ def shunxin_merge_final(kind, tags):
             pass
 
 
-def shunxin_expected_download(pages, foreground=True):
+def shunxin_expected_download(pages, foreground=True, force=False):
     """顺心：应到货物数据下载（双账号/双归属地，内部含异常兜底重试与数据融合）。
 
     pages 为该站点的 page 列表（双账号在同一窗口的各一个标签页）。
@@ -175,7 +175,9 @@ def shunxin_expected_download(pages, foreground=True):
         ok = with_retry(
             f"顺心-{tag}",
             "应到",
-            lambda p=pg, t=tag: shunxin_expected_download_impl(p, out_tag=t),
+            lambda p=pg, t=tag, f=force: shunxin_expected_download_impl(
+                p, out_tag=t, force=f
+            ),
             lambda p=pg: shunxin_reset(p),
         )
         if not ok:
@@ -185,7 +187,7 @@ def shunxin_expected_download(pages, foreground=True):
     return True
 
 
-def shunxin_expected_download_impl(page, out_tag=""):
+def shunxin_expected_download_impl(page, out_tag="", force=False):
     """顺心：应到货物数据下载（单次执行，无重试；供自动化测试探测原始结果用）。
 
     out_tag 为归属地标签时，合并产物命名为「顺心-{out_tag}-应到货物数据.xlsx」，
@@ -309,12 +311,44 @@ def shunxin_expected_download_impl(page, out_tag=""):
         count = waybill_btns.count()
         print(f">> 共发现 {count} 个班次需要导出。")
 
+        # 【去重】加载本站已落库交接单号；force=True 或查询失败时 existing=空集（不去重）。
+        # 两账号共享同一集合（班次号/交接单号跨归属地不重叠）。
+        if force:
+            existing = set()
+            print(">> [去重] 强制重下，跳过去重。")
+        else:
+            try:
+                from inbound_verify import store
+
+                existing = store.get_existing_handover_nos("顺心")
+            except Exception as _e:
+                existing = set()
+                print(f">> [去重] 加载已落库交接单号失败，本次不去重: {_e}")
+
         for i in range(count):
             print(f"   ⏳ 正在处理第 {i+1}/{count} 个班次...")
 
             waybill_btns.nth(i).click()
             page.locator("label[title='运单查询']").wait_for(state="visible")
 
+            # 【方式1】运单列表界面已加载，读交接单号（RTS 开头）→ 已落库则退回列表跳过。
+            #   交接单号格式 RTS\d{3}WJ\d+（如 RTS023WJ374837），用 [A-Z0-9]+ 连续匹配整段。
+            #   读不到（DOM 变动/未渲染）则 handover_no 为空 → 不跳过（安全降级，继续导出）。
+            handover_no = ""
+            try:
+                _txt = page.locator("text=/RTS\\d+/").first.inner_text(timeout=3000)
+                _m = re.search(r"RTS[A-Z0-9]+", _txt)
+                if _m:
+                    handover_no = _m.group(0)
+            except Exception:
+                pass
+            print(f"      -> 运单列表交接单号：{handover_no or '(未读到，不去重)'}")
+            if handover_no and handover_no in existing:
+                print(f"      ⏭️ 交接单号 {handover_no} 已落库，跳过提交导出。")
+                page.get_by_role("tab", name="车辆点到").click()
+                page.wait_for_timeout(500)
+                continue
+
             # 4. 执行导出流程
             page.get_by_role("button", name="export 导出").click()
 
@@ -342,6 +376,11 @@ def shunxin_expected_download_impl(page, out_tag=""):
         _close_tab(page, "运单列表")
         _close_tab(page, "车辆点到")
 
+        # 【去重兜底】全部已落库/无数据 → 无导出任务，标签页已关，跳过下载轮询
+        if not export_times:
+            print(">> 本次无新班次需导出（全部已落库或无数据），结束。")
+            return True
+
         # 6. 前往数据导出页面去下载
         print(">> 正在前往【数据导出】界面...")
         page.locator("a[href='/dataExport']").click()
@@ -477,7 +516,7 @@ def shunxin_expected_download_impl(page, out_tag=""):
         return False
 
 
-def shunxin_actual_download(pages, foreground=True):
+def shunxin_actual_download(pages, foreground=True, force=False):
     """顺心：实到货物数据下载（双账号/双归属地，内部含异常兜底重试与数据融合）。
 
     与 shunxin_expected_download 同构：读归属地 → 去重校验 → 顺序各账号下载 →

inbound_verify/sites/yunda.py

@@ -169,18 +169,18 @@ def yunda_smart_menu_click(page, menu_path):
             page.wait_for_timeout(1000)
 
 
-def yunda_expected_download(page):
+def yunda_expected_download(page, force=False):
     """韵达：应到货物数据下载（内部含异常兜底重试，路由层无感）。"""
 
     return with_retry(
         "韵达",
         "应到",
-        lambda: yunda_expected_download_impl(page),
+        lambda: yunda_expected_download_impl(page, force=force),
         lambda: yunda_reset(page),
     )
 
 
-def yunda_expected_download_impl(page):
+def yunda_expected_download_impl(page, force=False):
     """韵达：应到货物数据下载（单次执行，无重试；供自动化测试探测原始结果用）。"""
     print("\n▶ 开始执行【韵达 - 应到货物数据下载】任务...")
 
@@ -283,6 +283,19 @@ def yunda_expected_download_impl(page):
         row_count = main_rows.count()
         print(f">> 当前视窗共捕获到活跃交接单记录: {row_count} 条")
 
+        # 【去重】加载本站已落库交接单号；force=True 或查询失败时 existing=空集（不去重）
+        if force:
+            existing = set()
+            print(">> [去重] 强制重下，跳过去重。")
+        else:
+            try:
+                from inbound_verify import store
+
+                existing = store.get_existing_handover_nos("韵达")
+            except Exception as _e:
+                existing = set()
+                print(f">> [去重] 加载已落库交接单号失败，本次不去重: {_e}")
+
         # 5. 逐行双击并提交导出
         for i in range(row_count):
             print(f"   ⏳ 正在处理第 {i+1}/{row_count} 个交接单模块...")
@@ -290,6 +303,11 @@ def yunda_expected_download_impl(page):
 
             raw_no = current_row.locator("td").nth(1).inner_text().strip()
 
+            # 【去重】已落库的交接单号不再提交导出任务
+            if raw_no in existing:
+                print(f"      ⏭️ 交接单号 {raw_no} 已落库，跳过提交导出。")
+                continue
+
             # 跳过已绑定的交接单
             bind_status = current_row.locator("td").nth(2).inner_text().strip()
             print(f"      -> 交接单号: {raw_no} [绑定状态: {bind_status}]")
@@ -404,7 +422,7 @@ def yunda_expected_download_impl(page):
         return False
 
 
-def yunda_actual_download(page):
+def yunda_actual_download(page, force=False):
     """韵达：实到货物数据下载（内部含异常兜底重试，路由层无感）。"""
 
     return with_retry(

inbound_verify/sites/zto.py

@@ -109,18 +109,18 @@ def zto_smart_menu_click(page, menu_path):
             page.wait_for_timeout(1000)
 
 
-def zto_expected_download(page):
+def zto_expected_download(page, force=False):
     """中通：应到货物数据下载（内部含异常兜底重试，路由层无感）。"""
 
     return with_retry(
         "中通",
         "应到",
-        lambda: zto_expected_download_impl(page),
+        lambda: zto_expected_download_impl(page, force=force),
         lambda: zto_reset(page),
     )
 
 
-def zto_expected_download_impl(page):
+def zto_expected_download_impl(page, force=False):
     """中通：应到货物数据下载（单次执行，无重试；供自动化测试探测原始结果用）。"""
     print("\n▶ 开始执行【中通 - 应到货物数据下载】任务...")
 
@@ -240,6 +240,19 @@ def zto_expected_download_impl(page):
         count = main_rows.count()
         print(f">> 共发现 {count} 个交接单需要导出。")
 
+        # 【去重】加载本站已落库交接单号；force=True 或查询失败时 existing=空集（不去重）
+        if force:
+            existing = set()
+            print(">> [去重] 强制重下，跳过去重。")
+        else:
+            try:
+                from inbound_verify import store
+
+                existing = store.get_existing_handover_nos("中通")
+            except Exception as _e:
+                existing = set()
+                print(f">> [去重] 加载已落库交接单号失败，本次不去重: {_e}")
+
         for i in range(count):
             print(f"   ⏳ 正在处理第 {i+1}/{count} 个交接单...")
             row = ewb_frame.locator(
@@ -251,6 +264,11 @@ def zto_expected_download_impl(page):
             handover_no = match.group(0) if match else raw_text.strip()
             print(f"      -> 当前交接单号：{handover_no}")
 
+            # 【去重】已落库的交接单号不再提交导出任务（不双击、不 append export_times）
+            if handover_no in existing:
+                print(f"      ⏭️ 交接单号 {handover_no} 已落库，跳过提交导出。")
+                continue
+
             row.dblclick()
 
             ewb_frame.locator("#datagrid2").get_by_text("运单号").wait_for(
@@ -316,6 +334,11 @@ def zto_expected_download_impl(page):
         except Exception as e:
             print(f"   ⚠️ 关闭【进站交接单查询】标签页时出错: {e}")
 
+        # 【去重兜底】全部已落库/无数据 → 无导出任务，标签页已关，直接结束不进轮询
+        if not export_times:
+            print(">> 本次无新交接单需导出（全部已落库或无数据），结束。")
+            return True
+
         # 交由统一的轮询下载流程处理
         _zto_poll_and_download_tasks(
             page,
@@ -330,7 +353,7 @@ def zto_expected_download_impl(page):
         return False
 
 
-def zto_actual_download(page):
+def zto_actual_download(page, force=False):
     """中通：实到货物数据下载（内部含异常兜底重试，路由层无感）。"""
 
     return with_retry(

inbound_verify/store.py

@@ -422,6 +422,25 @@ def ingest_task(site, kind):
     return total
 
 
+def get_existing_handover_nos(site):
+    """查该站点已落库的交接单号集合（expected_record.handover_no）。
+    供"提交导出任务前"去重：已落库的交接单号不再重复提交导出任务。
+    PG 不可用（cpolar 抖动等）时返回空集 + 告警，调用方按"未确认存在"处理
+    （继续提交导出，UPSERT 兜底，绝不因去重查询失败而漏数据）。"""
+    try:
+        with _connect(_load_pg_config()["dbname"]) as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    "SELECT handover_no FROM expected_record "
+                    "WHERE site=%s AND handover_no IS NOT NULL AND handover_no <> ''",
+                    (site,),
+                )
+                return {str(r[0]).strip() for r in cur.fetchall()}
+    except Exception as e:
+        print(f">> [去重] 查询已落库交接单号失败（{site}），本次不去重: {e}")
+        return set()
+
+
 # ============================== 命令行 ==============================