修复 169 源文件瞬态导致的 “File is not a zip file”
根因:源 Excel 常被打开/保存,download 在拷贝瞬间抓到 0 字节或 残缺文件,openpyxl 解析即报 “File is not a zip file”;且 runner 在 download 之后、parse 之前就无条件 save_baseline,把 size=0 的瞬态 源记成 “已处理”,污染基线。 修复: - source_watcher.download:下载后校验(非空 + ZIP 头 PK),不合法 则删除并重试(默认 3 次,间隔 3s 等待源落盘)。 - source_watcher.has_changed:源不可达时优雅跳过本轮(WARNING), 不再每轮刷 traceback(169 间歇不可达为已知问题)。 - runner.run_once:save_baseline 移到整轮(解析+写库)成功之后, 杜绝把损坏/瞬态源记为已处理。
This commit is contained in:
@@ -38,17 +38,17 @@ def setup_logging(cfg):
|
||||
|
||||
|
||||
def run_once(cfg, source_path=None, dry_run=False):
|
||||
unc = cfg["source"]["unc_path"]
|
||||
cache = cfg["source"]["local_cache_dir"]
|
||||
meta_path = os.path.join(cache, ".source_meta.json")
|
||||
|
||||
if source_path:
|
||||
local_file = source_path
|
||||
logger.info("使用本地源: %s", local_file)
|
||||
else:
|
||||
unc = cfg["source"]["unc_path"]
|
||||
cache = cfg["source"]["local_cache_dir"]
|
||||
meta_path = os.path.join(cache, ".source_meta.json")
|
||||
if not source_watcher.has_changed(unc, meta_path):
|
||||
return
|
||||
local_file = source_watcher.download(unc, cache)
|
||||
source_watcher.save_baseline(meta_path, source_watcher.get_source_meta(unc))
|
||||
|
||||
records, fields = parse_excel(local_file, cfg.get("field_map", {}))
|
||||
|
||||
@@ -64,6 +64,14 @@ def run_once(cfg, source_path=None, dry_run=False):
|
||||
|
||||
sync_all(cfg, records, fields)
|
||||
|
||||
# 仅整轮(解析 + 写库)成功后才落基线,
|
||||
# 避免把瞬态/损坏源记为“已处理”(曾导致基线被记成 size=0)。
|
||||
if not source_path:
|
||||
try:
|
||||
source_watcher.save_baseline(meta_path, source_watcher.get_source_meta(unc))
|
||||
except OSError as e:
|
||||
logger.warning("保存基线失败(不影响本次同步): %s", e)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="采购执行情况即时数据 同步服务")
|
||||
|
||||
@@ -3,11 +3,34 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 源 Excel 常被前端用户打开/保存,拷贝瞬间可能抓到 0 字节或残缺文件
|
||||
# (表现为 openpyxl 报 “File is not a zip file”)。下载后校验并有限重试,
|
||||
# 等待源文件落盘后再取,规避竞态。
|
||||
_XLSX_RETRIES = 3
|
||||
_XLSX_RETRY_DELAY = 3.0
|
||||
|
||||
|
||||
def _is_valid_xlsx(path):
|
||||
"""轻量校验:非空 + ZIP 头(PK)。xlsx 本质是 zip 压缩包。"""
|
||||
try:
|
||||
sz = os.path.getsize(path)
|
||||
except OSError:
|
||||
return False
|
||||
if sz == 0:
|
||||
return False
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
head = f.read(4)
|
||||
except OSError:
|
||||
return False
|
||||
return head[:2] == b"PK" # PK\x03\x04
|
||||
|
||||
|
||||
def get_source_meta(path):
|
||||
st = os.stat(path)
|
||||
@@ -32,8 +55,10 @@ def has_changed(path, meta_path):
|
||||
try:
|
||||
cur = get_source_meta(path)
|
||||
except OSError as e:
|
||||
logger.error("无法访问源文件 %s: %s", path, e)
|
||||
raise
|
||||
# 169 共享间歇不可达 / 源正被 Excel 保存锁定:本轮跳过,
|
||||
# 下个周期再试,避免每轮刷一条 traceback。
|
||||
logger.warning("源文件暂时不可访问(可能正在保存/共享抖动),跳过本轮: %s", e)
|
||||
return False
|
||||
base = load_baseline(meta_path)
|
||||
if base is None:
|
||||
logger.info("无基准记录,视为有变化")
|
||||
@@ -43,11 +68,37 @@ def has_changed(path, meta_path):
|
||||
return changed
|
||||
|
||||
|
||||
def download(path, cache_dir):
|
||||
def download(path, cache_dir, retries=_XLSX_RETRIES, retry_delay=_XLSX_RETRY_DELAY):
|
||||
"""从 UNC 源复制 xlsx 到本地缓存,带校验 + 重试。
|
||||
|
||||
源文件常被 Excel 打开/保存,拷贝瞬间可能抓到 0 字节或残缺文件
|
||||
(openpyxl 随后报 “File is not a zip file”)。这里下载后做校验
|
||||
(非空 + ZIP 头 PK),不合法则删除并重试,间隔等待源落盘。
|
||||
"""
|
||||
cache = Path(cache_dir)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
dst = cache / f"{Path(path).stem}_{ts}.xlsx"
|
||||
shutil.copy2(path, str(dst))
|
||||
logger.info("已下载: %s -> %s", path, dst)
|
||||
return str(dst)
|
||||
last_err = None
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
shutil.copy2(path, str(dst))
|
||||
except OSError as e:
|
||||
last_err = e
|
||||
logger.warning("下载源文件失败(第%d/%d次): %s", attempt, retries, e)
|
||||
time.sleep(retry_delay)
|
||||
continue
|
||||
if not _is_valid_xlsx(dst):
|
||||
sz = os.path.getsize(dst) if os.path.exists(dst) else -1
|
||||
last_err = f"下载文件校验失败(非有效 xlsx,大小={sz})"
|
||||
logger.warning("下载文件校验失败(第%d/%d次): %s 大小=%d",
|
||||
attempt, retries, dst, max(sz, 0))
|
||||
try:
|
||||
os.remove(dst)
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(retry_delay)
|
||||
continue
|
||||
logger.info("已下载: %s -> %s", path, dst)
|
||||
return str(dst)
|
||||
raise RuntimeError(f"下载源文件失败(已重试{retries}次): {last_err}")
|
||||
|
||||
Reference in New Issue
Block a user