修复 169 源文件瞬态导致的 “File is not a zip file”
根因:源 Excel 常被打开/保存,download 在拷贝瞬间抓到 0 字节或 残缺文件,openpyxl 解析即报 “File is not a zip file”;且 runner 在 download 之后、parse 之前就无条件 save_baseline,把 size=0 的瞬态 源记成 “已处理”,污染基线。 修复: - source_watcher.download:下载后校验(非空 + ZIP 头 PK),不合法 则删除并重试(默认 3 次,间隔 3s 等待源落盘)。 - source_watcher.has_changed:源不可达时优雅跳过本轮(WARNING), 不再每轮刷 traceback(169 间歇不可达为已知问题)。 - runner.run_once:save_baseline 移到整轮(解析+写库)成功之后, 杜绝把损坏/瞬态源记为已处理。
This commit is contained in:
@@ -38,17 +38,17 @@ def setup_logging(cfg):
|
|||||||
|
|
||||||
|
|
||||||
def run_once(cfg, source_path=None, dry_run=False):
|
def run_once(cfg, source_path=None, dry_run=False):
|
||||||
|
unc = cfg["source"]["unc_path"]
|
||||||
|
cache = cfg["source"]["local_cache_dir"]
|
||||||
|
meta_path = os.path.join(cache, ".source_meta.json")
|
||||||
|
|
||||||
if source_path:
|
if source_path:
|
||||||
local_file = source_path
|
local_file = source_path
|
||||||
logger.info("使用本地源: %s", local_file)
|
logger.info("使用本地源: %s", local_file)
|
||||||
else:
|
else:
|
||||||
unc = cfg["source"]["unc_path"]
|
|
||||||
cache = cfg["source"]["local_cache_dir"]
|
|
||||||
meta_path = os.path.join(cache, ".source_meta.json")
|
|
||||||
if not source_watcher.has_changed(unc, meta_path):
|
if not source_watcher.has_changed(unc, meta_path):
|
||||||
return
|
return
|
||||||
local_file = source_watcher.download(unc, cache)
|
local_file = source_watcher.download(unc, cache)
|
||||||
source_watcher.save_baseline(meta_path, source_watcher.get_source_meta(unc))
|
|
||||||
|
|
||||||
records, fields = parse_excel(local_file, cfg.get("field_map", {}))
|
records, fields = parse_excel(local_file, cfg.get("field_map", {}))
|
||||||
|
|
||||||
@@ -64,6 +64,14 @@ def run_once(cfg, source_path=None, dry_run=False):
|
|||||||
|
|
||||||
sync_all(cfg, records, fields)
|
sync_all(cfg, records, fields)
|
||||||
|
|
||||||
|
# 仅整轮(解析 + 写库)成功后才落基线,
|
||||||
|
# 避免把瞬态/损坏源记为“已处理”(曾导致基线被记成 size=0)。
|
||||||
|
if not source_path:
|
||||||
|
try:
|
||||||
|
source_watcher.save_baseline(meta_path, source_watcher.get_source_meta(unc))
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning("保存基线失败(不影响本次同步): %s", e)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description="采购执行情况即时数据 同步服务")
|
parser = argparse.ArgumentParser(description="采购执行情况即时数据 同步服务")
|
||||||
|
|||||||
@@ -3,11 +3,34 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 源 Excel 常被前端用户打开/保存,拷贝瞬间可能抓到 0 字节或残缺文件
|
||||||
|
# (表现为 openpyxl 报 “File is not a zip file”)。下载后校验并有限重试,
|
||||||
|
# 等待源文件落盘后再取,规避竞态。
|
||||||
|
_XLSX_RETRIES = 3
|
||||||
|
_XLSX_RETRY_DELAY = 3.0
|
||||||
|
|
||||||
|
|
||||||
|
def _is_valid_xlsx(path):
|
||||||
|
"""轻量校验:非空 + ZIP 头(PK)。xlsx 本质是 zip 压缩包。"""
|
||||||
|
try:
|
||||||
|
sz = os.path.getsize(path)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
if sz == 0:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
head = f.read(4)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return head[:2] == b"PK" # PK\x03\x04
|
||||||
|
|
||||||
|
|
||||||
def get_source_meta(path):
|
def get_source_meta(path):
|
||||||
st = os.stat(path)
|
st = os.stat(path)
|
||||||
@@ -32,8 +55,10 @@ def has_changed(path, meta_path):
|
|||||||
try:
|
try:
|
||||||
cur = get_source_meta(path)
|
cur = get_source_meta(path)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
logger.error("无法访问源文件 %s: %s", path, e)
|
# 169 共享间歇不可达 / 源正被 Excel 保存锁定:本轮跳过,
|
||||||
raise
|
# 下个周期再试,避免每轮刷一条 traceback。
|
||||||
|
logger.warning("源文件暂时不可访问(可能正在保存/共享抖动),跳过本轮: %s", e)
|
||||||
|
return False
|
||||||
base = load_baseline(meta_path)
|
base = load_baseline(meta_path)
|
||||||
if base is None:
|
if base is None:
|
||||||
logger.info("无基准记录,视为有变化")
|
logger.info("无基准记录,视为有变化")
|
||||||
@@ -43,11 +68,37 @@ def has_changed(path, meta_path):
|
|||||||
return changed
|
return changed
|
||||||
|
|
||||||
|
|
||||||
def download(path, cache_dir):
|
def download(path, cache_dir, retries=_XLSX_RETRIES, retry_delay=_XLSX_RETRY_DELAY):
|
||||||
|
"""从 UNC 源复制 xlsx 到本地缓存,带校验 + 重试。
|
||||||
|
|
||||||
|
源文件常被 Excel 打开/保存,拷贝瞬间可能抓到 0 字节或残缺文件
|
||||||
|
(openpyxl 随后报 “File is not a zip file”)。这里下载后做校验
|
||||||
|
(非空 + ZIP 头 PK),不合法则删除并重试,间隔等待源落盘。
|
||||||
|
"""
|
||||||
cache = Path(cache_dir)
|
cache = Path(cache_dir)
|
||||||
cache.mkdir(parents=True, exist_ok=True)
|
cache.mkdir(parents=True, exist_ok=True)
|
||||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
dst = cache / f"{Path(path).stem}_{ts}.xlsx"
|
dst = cache / f"{Path(path).stem}_{ts}.xlsx"
|
||||||
shutil.copy2(path, str(dst))
|
last_err = None
|
||||||
logger.info("已下载: %s -> %s", path, dst)
|
for attempt in range(1, retries + 1):
|
||||||
return str(dst)
|
try:
|
||||||
|
shutil.copy2(path, str(dst))
|
||||||
|
except OSError as e:
|
||||||
|
last_err = e
|
||||||
|
logger.warning("下载源文件失败(第%d/%d次): %s", attempt, retries, e)
|
||||||
|
time.sleep(retry_delay)
|
||||||
|
continue
|
||||||
|
if not _is_valid_xlsx(dst):
|
||||||
|
sz = os.path.getsize(dst) if os.path.exists(dst) else -1
|
||||||
|
last_err = f"下载文件校验失败(非有效 xlsx,大小={sz})"
|
||||||
|
logger.warning("下载文件校验失败(第%d/%d次): %s 大小=%d",
|
||||||
|
attempt, retries, dst, max(sz, 0))
|
||||||
|
try:
|
||||||
|
os.remove(dst)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
time.sleep(retry_delay)
|
||||||
|
continue
|
||||||
|
logger.info("已下载: %s -> %s", path, dst)
|
||||||
|
return str(dst)
|
||||||
|
raise RuntimeError(f"下载源文件失败(已重试{retries}次): {last_err}")
|
||||||
|
|||||||
Reference in New Issue
Block a user