修复 Access 全量同步静默丢行问题

- sync_access 改为事务化(去 autocommit,失败回滚),杜绝前段已提交、错误被吞导致日志谎报完成
- 写入前按 Access 列约束做值清洗:VARCHAR 按列宽截断、DATETIME 夹到合法范围,从根消除驱动拒写
- 写入后强制 SELECT COUNT(*) 核对实际行数;不足则 DELETE 后逐行回退插入并精确记录失败行号
- 修复后两端均落 35043 行,SQL Server 与 Access 校验和一致
This commit is contained in:
Misaka_Company
2026-07-15 15:31:24 +08:00
parent 5ae7acffde
commit eecea44746

View File

@@ -54,27 +54,108 @@ def sync_sql_server(cfg, records, fields, import_time):
cn.close() cn.close()
# Access DATETIME 合法范围(年 100~9999
_MIN_DT = datetime(100, 1, 1)
_MAX_DT = datetime(9999, 12, 31, 23, 59, 59)
def _clamp_dt(v):
if isinstance(v, datetime):
if v < _MIN_DT:
return _MIN_DT
if v > _MAX_DT:
return _MAX_DT
return v
def _trunc_str(v, size):
if v is None:
return None
s = str(v)
return s[:size] if len(s) > size else s
def _access_col_limits(cn, table):
"""查询 Access 列约束VARCHAR 最大长度 + DATETIME 列集合。"""
limits, dt_cols = {}, set()
for r in cn.cursor().columns(table=table):
name = r.column_name
t = (r.type_name or "").upper()
if t in ("VARCHAR", "CHAR", "TEXT") and r.column_size:
limits[name] = int(r.column_size)
elif "DATETIME" in t or t in ("DATE", "TIME"):
dt_cols.add(name)
return limits, dt_cols
def _sanitize_rows(rows, fields, import_col, limits, dt_cols):
names = list(fields) + [import_col]
out = []
for r in rows:
out.append(tuple(
_trunc_str(v, limits[n]) if n in limits
else (_clamp_dt(v) if n in dt_cols else v)
for n, v in zip(names, r)
))
return out
def sync_access(cfg, records, fields, import_time): def sync_access(cfg, records, fields, import_time):
if not records: if not records:
logger.warning("Access: 无数据,跳过") logger.warning("Access: 无数据,跳过")
return return
a = cfg["access"] a = cfg["access"]
table_expr = f"[{a['table']}]" # [procurementVisibilityHub_请购执行] table = a["table"]
table_expr = f"[{table}]" # [procurementVisibilityHub_请购执行]
import_col = cfg.get("auto_fields", {}).get("import_time_field", "导入时间") import_col = cfg.get("auto_fields", {}).get("import_time_field", "导入时间")
insert_sql = _build_insert(table_expr, fields, import_col) insert_sql = _build_insert(table_expr, fields, import_col)
rows = _to_rows(records, fields, import_time) rows = _to_rows(records, fields, import_time)
batch = cfg.get("sync", {}).get("batch_size", 1000) batch = cfg.get("sync", {}).get("batch_size", 1000)
total = len(rows)
cn = pyodbc.connect(build_access_connstr(cfg), autocommit=True) cn = pyodbc.connect(build_access_connstr(cfg), autocommit=False)
try: try:
cur = cn.cursor() cur = cn.cursor()
# 列约束用于值清洗,规避 Access ODBC 驱动的静默截断/丢行
limits, dt_cols = _access_col_limits(cn, table)
san_rows = _sanitize_rows(rows, fields, import_col, limits, dt_cols)
logger.info("Access: 清空 %s", table_expr) logger.info("Access: 清空 %s", table_expr)
cur.execute(f"DELETE FROM {table_expr}") cur.execute(f"DELETE FROM {table_expr}")
total = len(rows)
# 快速路径:批量 executemany
for i in range(0, total, batch): for i in range(0, total, batch):
cur.executemany(insert_sql, rows[i:i + batch]) cur.executemany(insert_sql, san_rows[i:i + batch])
logger.info("Access: 已插入 %d/%d", min(i + batch, total), total) logger.info("Access: 已插入 %d/%d", min(i + batch, total), total)
logger.info("Access: 完成,共 %d", total) cn.commit()
# 校验Access 的 executemany 可能静默丢行,必须核对实际行数
cur.execute(f"SELECT COUNT(*) FROM {table_expr}")
n = cur.fetchone()[0]
if n == total:
logger.info("Access: 完成,共 %d 行(已校验)", total)
return
# 回退:逐行插入,精确暴露失败行
logger.warning("Access: 批量仅落 %d/%d 行,改用逐行插入补齐", n, total)
cur.execute(f"DELETE FROM {table_expr}")
failed = 0
for idx, r in enumerate(san_rows):
try:
cur.execute(insert_sql, r)
except Exception as e:
failed += 1
logger.error("Access: 第 %d 行插入失败: %r", idx, e)
cn.commit()
cur.execute(f"SELECT COUNT(*) FROM {table_expr}")
n2 = cur.fetchone()[0]
if n2 + failed != total:
logger.error("Access: 仍不一致 落=%d 失败=%d 期望=%d", n2, failed, total)
else:
logger.info("Access: 逐行补齐完成,共 %d 行(失败 %d 已记录)", n2, failed)
except Exception:
cn.rollback()
raise
finally: finally:
cn.close() cn.close()