修复 169 源文件瞬态导致的 “File is not a zip file”

根因:源 Excel 常被打开/保存,download 在拷贝瞬间抓到 0 字节或
残缺文件,openpyxl 解析即报 “File is not a zip file”;且 runner 在
download 之后、parse 之前就无条件 save_baseline,把 size=0 的瞬态
源记成 “已处理”,污染基线。

修复:
- source_watcher.download:下载后校验(非空 + ZIP 头 PK),不合法
  则删除并重试(默认 3 次,间隔 3s 等待源落盘)。
- source_watcher.has_changed:源不可达时优雅跳过本轮(WARNING),
  不再每轮刷 traceback(169 间歇不可达为已知问题)。
- runner.run_once:save_baseline 移到整轮(解析+写库)成功之后,
  杜绝把损坏/瞬态源记为已处理。
This commit is contained in:
Misaka_Company
2026-07-15 16:53:37 +08:00
parent 85ae96557e
commit 5e3c5d5b31
2 changed files with 69 additions and 10 deletions

View File

@@ -38,17 +38,17 @@ def setup_logging(cfg):
def run_once(cfg, source_path=None, dry_run=False): def run_once(cfg, source_path=None, dry_run=False):
unc = cfg["source"]["unc_path"]
cache = cfg["source"]["local_cache_dir"]
meta_path = os.path.join(cache, ".source_meta.json")
if source_path: if source_path:
local_file = source_path local_file = source_path
logger.info("使用本地源: %s", local_file) logger.info("使用本地源: %s", local_file)
else: else:
unc = cfg["source"]["unc_path"]
cache = cfg["source"]["local_cache_dir"]
meta_path = os.path.join(cache, ".source_meta.json")
if not source_watcher.has_changed(unc, meta_path): if not source_watcher.has_changed(unc, meta_path):
return return
local_file = source_watcher.download(unc, cache) local_file = source_watcher.download(unc, cache)
source_watcher.save_baseline(meta_path, source_watcher.get_source_meta(unc))
records, fields = parse_excel(local_file, cfg.get("field_map", {})) records, fields = parse_excel(local_file, cfg.get("field_map", {}))
@@ -64,6 +64,14 @@ def run_once(cfg, source_path=None, dry_run=False):
sync_all(cfg, records, fields) sync_all(cfg, records, fields)
# 仅整轮(解析 + 写库)成功后才落基线,
# 避免把瞬态/损坏源记为“已处理”(曾导致基线被记成 size=0
if not source_path:
try:
source_watcher.save_baseline(meta_path, source_watcher.get_source_meta(unc))
except OSError as e:
logger.warning("保存基线失败(不影响本次同步): %s", e)
def main(): def main():
parser = argparse.ArgumentParser(description="采购执行情况即时数据 同步服务") parser = argparse.ArgumentParser(description="采购执行情况即时数据 同步服务")

View File

@@ -3,11 +3,34 @@ import json
import logging import logging
import os import os
import shutil import shutil
import time
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# 源 Excel 常被前端用户打开/保存,拷贝瞬间可能抓到 0 字节或残缺文件
# (表现为 openpyxl 报 “File is not a zip file”。下载后校验并有限重试
# 等待源文件落盘后再取,规避竞态。
_XLSX_RETRIES = 3
_XLSX_RETRY_DELAY = 3.0
def _is_valid_xlsx(path):
"""轻量校验:非空 + ZIP 头(PK)。xlsx 本质是 zip 压缩包。"""
try:
sz = os.path.getsize(path)
except OSError:
return False
if sz == 0:
return False
try:
with open(path, "rb") as f:
head = f.read(4)
except OSError:
return False
return head[:2] == b"PK" # PK\x03\x04
def get_source_meta(path): def get_source_meta(path):
st = os.stat(path) st = os.stat(path)
@@ -32,8 +55,10 @@ def has_changed(path, meta_path):
try: try:
cur = get_source_meta(path) cur = get_source_meta(path)
except OSError as e: except OSError as e:
logger.error("无法访问源文件 %s: %s", path, e) # 169 共享间歇不可达 / 源正被 Excel 保存锁定:本轮跳过,
raise # 下个周期再试,避免每轮刷一条 traceback。
logger.warning("源文件暂时不可访问(可能正在保存/共享抖动),跳过本轮: %s", e)
return False
base = load_baseline(meta_path) base = load_baseline(meta_path)
if base is None: if base is None:
logger.info("无基准记录,视为有变化") logger.info("无基准记录,视为有变化")
@@ -43,11 +68,37 @@ def has_changed(path, meta_path):
return changed return changed
def download(path, cache_dir): def download(path, cache_dir, retries=_XLSX_RETRIES, retry_delay=_XLSX_RETRY_DELAY):
"""从 UNC 源复制 xlsx 到本地缓存,带校验 + 重试。
源文件常被 Excel 打开/保存,拷贝瞬间可能抓到 0 字节或残缺文件
openpyxl 随后报 “File is not a zip file”。这里下载后做校验
(非空 + ZIP 头 PK不合法则删除并重试间隔等待源落盘。
"""
cache = Path(cache_dir) cache = Path(cache_dir)
cache.mkdir(parents=True, exist_ok=True) cache.mkdir(parents=True, exist_ok=True)
ts = datetime.now().strftime("%Y%m%d_%H%M%S") ts = datetime.now().strftime("%Y%m%d_%H%M%S")
dst = cache / f"{Path(path).stem}_{ts}.xlsx" dst = cache / f"{Path(path).stem}_{ts}.xlsx"
shutil.copy2(path, str(dst)) last_err = None
logger.info("已下载: %s -> %s", path, dst) for attempt in range(1, retries + 1):
return str(dst) try:
shutil.copy2(path, str(dst))
except OSError as e:
last_err = e
logger.warning("下载源文件失败(第%d/%d次): %s", attempt, retries, e)
time.sleep(retry_delay)
continue
if not _is_valid_xlsx(dst):
sz = os.path.getsize(dst) if os.path.exists(dst) else -1
last_err = f"下载文件校验失败(非有效 xlsx大小={sz})"
logger.warning("下载文件校验失败(第%d/%d次): %s 大小=%d",
attempt, retries, dst, max(sz, 0))
try:
os.remove(dst)
except OSError:
pass
time.sleep(retry_delay)
continue
logger.info("已下载: %s -> %s", path, dst)
return str(dst)
raise RuntimeError(f"下载源文件失败(已重试{retries}次): {last_err}")