83c90f0161
- Consolidated database, file source, and field mapping configurations into dedicated modules under the `config` directory. - Removed hardcoded database connection details from `migration.py` and replaced them with imports from the new configuration structure. - Updated `ntfy_utils.py` and `run_incremental_sync.py` to utilize the new configuration imports for cleaner code and better maintainability. - Deleted `update_config.py` as its contents have been integrated into the new configuration files. - Added a new `settings.local.json` for managing permissions related to script execution. - Enhanced the structure of the migration tasks and Excel configurations for better organization and clarity.
284 lines
11 KiB
Python
284 lines
11 KiB
Python
import time
|
||
import os
|
||
import sys
|
||
import pyodbc
|
||
from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, POLL_INTERVAL, LOG_TABLE_CONFIG
|
||
import db_utils
|
||
import logging
|
||
from logging.handlers import TimedRotatingFileHandler
|
||
import ntfy_utils
|
||
|
||
# ================= 日志系统配置 =================
|
||
def setup_logger():
|
||
log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'log')
|
||
if not os.path.exists(log_dir):
|
||
os.makedirs(log_dir)
|
||
|
||
logger = logging.getLogger("SyncService")
|
||
logger.setLevel(logging.INFO)
|
||
logger.handlers = []
|
||
|
||
log_file_path = os.path.join(log_dir, 'sync_service.log')
|
||
file_handler = TimedRotatingFileHandler(
|
||
log_file_path, when='midnight', interval=1, backupCount=30, encoding='utf-8'
|
||
)
|
||
file_handler.suffix = "%Y-%m-%d"
|
||
file_fmt = logging.Formatter('%(asctime)s - [%(levelname)s] - %(message)s')
|
||
file_handler.setFormatter(file_fmt)
|
||
|
||
console_handler = logging.StreamHandler(sys.stdout)
|
||
console_fmt = logging.Formatter('%(asctime)s - %(message)s', datefmt='%H:%M:%S')
|
||
console_handler.setFormatter(console_fmt)
|
||
|
||
logger.addHandler(file_handler)
|
||
logger.addHandler(console_handler)
|
||
return logger
|
||
|
||
logger = setup_logger()
|
||
|
||
# ================= 日志辅助函数 =================
|
||
def log_success(message):
|
||
"""成功消息 - 绿色感觉"""
|
||
logger.info(f"✅ [成功] {message}")
|
||
|
||
def log_error(message):
|
||
"""错误消息 - 红色感觉"""
|
||
logger.error(f"❌ [错误] {message}")
|
||
ntfy_utils.send_error(f"❌ [错误] {message}")
|
||
|
||
def log_warning(message):
|
||
"""警告消息 - 黄色感觉"""
|
||
logger.warning(f"⚠️ [警告] {message}")
|
||
ntfy_utils.send_error(f"⚠️ [警告] {message}")
|
||
|
||
def log_info(message):
|
||
"""信息消息"""
|
||
logger.info(f"ℹ️ {message}")
|
||
|
||
def log_processing(message):
|
||
"""处理中消息"""
|
||
logger.info(f"🔄 [处理] {message}")
|
||
|
||
def log_skip(message):
|
||
"""跳过消息"""
|
||
logger.info(f"⏭️ [跳过] {message}")
|
||
|
||
def log_critical(message):
|
||
"""严重错误"""
|
||
logger.critical(f"🔥 [严重] {message}")
|
||
ntfy_utils.send_critical(f"🔥 [严重] {message}")
|
||
|
||
def log_start(message):
|
||
"""启动消息"""
|
||
logger.info(f"🚀 [启动] {message}")
|
||
ntfy_utils.send_ntfy(f"🚀 [启动] {message}")
|
||
|
||
def log_stop(message):
|
||
"""停止消息"""
|
||
logger.info(f"🛑 [停止] {message}")
|
||
ntfy_utils.send_ntfy(f"🛑 [停止] {message}")
|
||
|
||
def log_file(message):
|
||
"""文件操作消息"""
|
||
logger.info(f"📂 [文件] {message}")
|
||
|
||
def log_database(message):
|
||
"""数据库操作消息"""
|
||
logger.info(f"💾 [数据库] {message}")
|
||
|
||
def log_sync(message):
|
||
"""同步操作消息"""
|
||
logger.info(f"🔃 [同步] {message}")
|
||
|
||
# ================= 主逻辑 =================
|
||
|
||
def process_sync_task():
|
||
"""
|
||
逻辑重构：
|
||
遍历 SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。
|
||
"""
|
||
sql_conn = db_utils.get_sql_conn()
|
||
sql_cursor = sql_conn.cursor()
|
||
sql_cursor.fast_executemany = True # 高性能开关
|
||
|
||
# 标记是否有工作被处理（用于控制轮询休眠时间）
|
||
work_done = False
|
||
|
||
cols = LOG_TABLE_CONFIG
|
||
log_full_name = db_utils.fmt_table(cols['schema'], cols['table_name'])
|
||
|
||
try:
|
||
# === 核心循环：以配置文件为驱动 ===
|
||
for clean_path, tables_map in SYNC_MAPPING.items():
|
||
|
||
# 1. 准备查询条件
|
||
# 获取该文件下所有需要同步的表名列表
|
||
target_tables = list(tables_map.keys())
|
||
if not target_tables:
|
||
continue
|
||
|
||
# 构造 TableAddress 的精确匹配条件
|
||
# VBA 逻辑：网络路径带 ";DATABASE=", 本地路径带 "LOCAL:"
|
||
# 我们直接构造这两个字符串，让 SQL Server 做精确匹配，效率极高
|
||
# 注意：将路径转为 Windows 标准反斜杠
|
||
win_path = os.path.normpath(clean_path)
|
||
addr_candidates = [
|
||
f";DATABASE={win_path}", # 情况1
|
||
f"LOCAL={win_path}", # 情况2 (注意 VBA 代码里可能是 LOCAL: 或 LOCAL=，请核对)
|
||
f"LOCAL:{win_path}", # 情况3
|
||
win_path # 情况4 (兼容没有前缀的情况)
|
||
]
|
||
|
||
# 2. 构造动态 SQL 查询
|
||
# WHERE Synced=0 AND Address IN (...) AND TableName IN (...)
|
||
placeholders_addr = ','.join(['?'] * len(addr_candidates))
|
||
placeholders_tbl = ','.join(['?'] * len(target_tables))
|
||
|
||
query_log = f"""
|
||
SELECT TOP 1000
|
||
{cols['col_log_id']},
|
||
{cols['col_table_name']},
|
||
{cols['col_record_id']}
|
||
FROM {log_full_name}
|
||
WHERE {cols['col_synced']} = 0
|
||
AND TableType = 'LINKED_ACCESS'
|
||
AND {cols['col_address']} IN ({placeholders_addr})
|
||
AND {cols['col_table_name']} IN ({placeholders_tbl})
|
||
ORDER BY {cols['col_log_id']} ASC
|
||
"""
|
||
|
||
# 参数列表：先放地址，再放表名
|
||
params = addr_candidates + target_tables
|
||
|
||
sql_cursor.execute(query_log, params)
|
||
logs = sql_cursor.fetchall()
|
||
|
||
if not logs:
|
||
continue # 这个文件没有需要同步的记录，检查下一个文件
|
||
|
||
work_done = True # 标记有工作
|
||
log_file(f"{os.path.basename(clean_path)} 发现 {len(logs)} 条待同步变更")
|
||
|
||
# 3. 本地分组 (按表名)
|
||
# 结构: table_tasks[TableName] = { ids: {}, log_ids: [] }
|
||
table_tasks = {}
|
||
for row in logs:
|
||
log_id, acc_table, record_id = row
|
||
if acc_table not in table_tasks:
|
||
table_tasks[acc_table] = {'record_ids': set(), 'log_ids': []}
|
||
table_tasks[acc_table]['record_ids'].add(record_id)
|
||
table_tasks[acc_table]['log_ids'].append(log_id)
|
||
|
||
# 4. 执行同步 (连接一次 Access，处理多张表)
|
||
if not os.path.exists(clean_path):
|
||
log_error(f"无法访问文件: {clean_path}")
|
||
continue
|
||
|
||
try:
|
||
acc_conn = db_utils.get_access_conn(clean_path)
|
||
acc_cursor = acc_conn.cursor()
|
||
log_database(f"已连接 Access 文件: {os.path.basename(clean_path)}")
|
||
except Exception as conn_err:
|
||
log_error(f"连接 Access 失败 [{os.path.basename(clean_path)}]: {conn_err}")
|
||
continue
|
||
|
||
# 统计每个文件的同步情况
|
||
file_success_count = 0
|
||
file_error_count = 0
|
||
file_total_records = 0
|
||
|
||
for acc_table, data in table_tasks.items():
|
||
record_ids = list(data['record_ids'])
|
||
log_ids = data['log_ids']
|
||
|
||
# 读取目标配置
|
||
target_conf = tables_map[acc_table]
|
||
target_schema = target_conf['target_schema']
|
||
target_table = target_conf['target_table']
|
||
pk_col = target_conf['pk_col']
|
||
target_full_name = db_utils.fmt_table(target_schema, target_table)
|
||
|
||
log_processing(f"正在同步表 [{acc_table}] → [{target_table}] ({len(record_ids)} 条记录)")
|
||
|
||
try:
|
||
# --- A. Access 查新数据 ---
|
||
ids_placeholders = ','.join(['?'] * len(record_ids))
|
||
acc_sql = f"SELECT * FROM [{acc_table}] WHERE [{pk_col}] IN ({ids_placeholders})"
|
||
acc_cursor.execute(acc_sql, record_ids)
|
||
new_rows = acc_cursor.fetchall()
|
||
acc_cols = [col[0] for col in acc_cursor.description]
|
||
|
||
# --- B. SQL Server 删旧插新 (事务) ---
|
||
# 1. 删除
|
||
del_sql = f"DELETE FROM {target_full_name} WHERE [{pk_col}] IN ({ids_placeholders})"
|
||
sql_cursor.execute(del_sql, record_ids)
|
||
|
||
# 2. 插入
|
||
if new_rows:
|
||
insert_sql = db_utils.generate_insert_sql(target_schema, target_table, acc_cols)
|
||
try: sql_cursor.execute(f"SET IDENTITY_INSERT {target_full_name} ON")
|
||
except: pass
|
||
|
||
sql_cursor.executemany(insert_sql, new_rows)
|
||
|
||
try: sql_cursor.execute(f"SET IDENTITY_INSERT {target_full_name} OFF")
|
||
except: pass
|
||
|
||
# 3. 标记日志 Synced = 1
|
||
log_placeholders = ','.join(['?'] * len(log_ids))
|
||
update_log_sql = f"""
|
||
UPDATE {log_full_name}
|
||
SET {cols['col_synced']} = 1
|
||
WHERE {cols['col_log_id']} IN ({log_placeholders})
|
||
"""
|
||
sql_cursor.execute(update_log_sql, log_ids)
|
||
|
||
sql_conn.commit()
|
||
log_success(f"表 [{target_table}] 同步完成: {len(record_ids)} 条记录")
|
||
|
||
file_success_count += 1
|
||
file_total_records += len(record_ids)
|
||
|
||
except Exception as tbl_err:
|
||
log_error(f"表 [{acc_table}] 同步失败: {tbl_err}")
|
||
sql_conn.rollback()
|
||
file_error_count += 1
|
||
|
||
acc_conn.close() # 关闭 Access 连接
|
||
|
||
# 输出文件级别的汇总
|
||
if file_success_count > 0 or file_error_count > 0:
|
||
summary = f"文件 [{os.path.basename(clean_path)}] 同步汇总: "
|
||
summary += f"成功 {file_success_count} 张表 ({file_total_records} 条记录)"
|
||
if file_error_count > 0:
|
||
summary += f" | 失败 {file_error_count} 张表"
|
||
log_sync(summary)
|
||
|
||
return work_done
|
||
|
||
except Exception as e:
|
||
log_critical(f"全局异常: {e}")
|
||
return False
|
||
finally:
|
||
try: sql_conn.close()
|
||
except: pass
|
||
|
||
if __name__ == "__main__":
|
||
log_start("增量同步服务已启动 (配置驱动模式)")
|
||
log_info(f"轮询间隔: {POLL_INTERVAL} 秒")
|
||
log_info(f"监控配置: {len(SYNC_MAPPING)} 个文件")
|
||
logger.info("=" * 70)
|
||
|
||
while True:
|
||
try:
|
||
has_work = process_sync_task()
|
||
# 如果有工作，说明可能还有积压，休息短一点(0.1s)
|
||
# 如果没工作，休息标准间隔(5s)
|
||
time.sleep(0.1 if has_work else POLL_INTERVAL)
|
||
except KeyboardInterrupt:
|
||
logger.info("=" * 70)
|
||
log_stop("收到停止信号，服务正在关闭...")
|
||
break
|
||
except Exception as e:
|
||
log_critical(f"主循环崩溃: {e}")
|
||
time.sleep(5) |