import time import os import sys import pyodbc import config import db_utils import logging from logging.handlers import TimedRotatingFileHandler import ntfy_utils # ================= 日志系统配置 ================= def setup_logger(): log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'log') if not os.path.exists(log_dir): os.makedirs(log_dir) logger = logging.getLogger("SyncService") logger.setLevel(logging.INFO) logger.handlers = [] log_file_path = os.path.join(log_dir, 'sync_service.log') file_handler = TimedRotatingFileHandler( log_file_path, when='midnight', interval=1, backupCount=30, encoding='utf-8' ) file_handler.suffix = "%Y-%m-%d" file_fmt = logging.Formatter('%(asctime)s - [%(levelname)s] - %(message)s') file_handler.setFormatter(file_fmt) console_handler = logging.StreamHandler(sys.stdout) console_fmt = logging.Formatter('%(asctime)s - %(message)s', datefmt='%H:%M:%S') console_handler.setFormatter(console_fmt) logger.addHandler(file_handler) logger.addHandler(console_handler) return logger logger = setup_logger() # ================= 日志辅助函数 ================= def log_success(message): """成功消息 - 绿色感觉""" logger.info(f"✅ [成功] {message}") def log_error(message): """错误消息 - 红色感觉""" logger.error(f"❌ [错误] {message}") ntfy_utils.send_error(f"❌ [错误] {message}") def log_warning(message): """警告消息 - 黄色感觉""" logger.warning(f"⚠️ [警告] {message}") ntfy_utils.send_error(f"⚠️ [警告] {message}") def log_info(message): """信息消息""" logger.info(f"ℹ️ {message}") def log_processing(message): """处理中消息""" logger.info(f"🔄 [处理] {message}") def log_skip(message): """跳过消息""" logger.info(f"⏭️ [跳过] {message}") def log_critical(message): """严重错误""" logger.critical(f"🔥 [严重] {message}") ntfy_utils.send_critical(f"🔥 [严重] {message}") def log_start(message): """启动消息""" logger.info(f"🚀 [启动] {message}") ntfy_utils.send_ntfy(f"🚀 [启动] {message}") def log_stop(message): """停止消息""" logger.info(f"🛑 [停止] {message}") ntfy_utils.send_ntfy(f"🛑 [停止] {message}") def log_file(message): """文件操作消息""" logger.info(f"📂 [文件] {message}") def log_database(message): """数据库操作消息""" logger.info(f"💾 [数据库] {message}") def log_sync(message): """同步操作消息""" logger.info(f"🔃 [同步] {message}") # ================= 主逻辑 ================= def process_sync_task(): """ 逻辑重构: 遍历 config.SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。 """ sql_conn = db_utils.get_sql_conn() sql_cursor = sql_conn.cursor() sql_cursor.fast_executemany = True # 高性能开关 # 标记是否有工作被处理(用于控制轮询休眠时间) work_done = False cols = config.LOG_TABLE_CONFIG log_full_name = db_utils.fmt_table(cols['schema'], cols['table_name']) try: # === 核心循环:以配置文件为驱动 === for clean_path, tables_map in config.SYNC_MAPPING.items(): # 1. 准备查询条件 # 获取该文件下所有需要同步的表名列表 target_tables = list(tables_map.keys()) if not target_tables: continue # 构造 TableAddress 的精确匹配条件 # VBA 逻辑:网络路径带 ";DATABASE=", 本地路径带 "LOCAL:" # 我们直接构造这两个字符串,让 SQL Server 做精确匹配,效率极高 # 注意:将路径转为 Windows 标准反斜杠 win_path = os.path.normpath(clean_path) addr_candidates = [ f";DATABASE={win_path}", # 情况1 f"LOCAL={win_path}", # 情况2 (注意 VBA 代码里可能是 LOCAL: 或 LOCAL=,请核对) f"LOCAL:{win_path}", # 情况3 win_path # 情况4 (兼容没有前缀的情况) ] # 2. 构造动态 SQL 查询 # WHERE Synced=0 AND Address IN (...) AND TableName IN (...) placeholders_addr = ','.join(['?'] * len(addr_candidates)) placeholders_tbl = ','.join(['?'] * len(target_tables)) query_log = f""" SELECT TOP 1000 {cols['col_log_id']}, {cols['col_table_name']}, {cols['col_record_id']} FROM {log_full_name} WHERE {cols['col_synced']} = 0 AND TableType = 'LINKED_ACCESS' AND {cols['col_address']} IN ({placeholders_addr}) AND {cols['col_table_name']} IN ({placeholders_tbl}) ORDER BY {cols['col_log_id']} ASC """ # 参数列表:先放地址,再放表名 params = addr_candidates + target_tables sql_cursor.execute(query_log, params) logs = sql_cursor.fetchall() if not logs: continue # 这个文件没有需要同步的记录,检查下一个文件 work_done = True # 标记有工作 log_file(f"{os.path.basename(clean_path)} 发现 {len(logs)} 条待同步变更") # 3. 本地分组 (按表名) # 结构: table_tasks[TableName] = { ids: {}, log_ids: [] } table_tasks = {} for row in logs: log_id, acc_table, record_id = row if acc_table not in table_tasks: table_tasks[acc_table] = {'record_ids': set(), 'log_ids': []} table_tasks[acc_table]['record_ids'].add(record_id) table_tasks[acc_table]['log_ids'].append(log_id) # 4. 执行同步 (连接一次 Access,处理多张表) if not os.path.exists(clean_path): log_error(f"无法访问文件: {clean_path}") continue try: acc_conn = db_utils.get_access_conn(clean_path) acc_cursor = acc_conn.cursor() log_database(f"已连接 Access 文件: {os.path.basename(clean_path)}") except Exception as conn_err: log_error(f"连接 Access 失败 [{os.path.basename(clean_path)}]: {conn_err}") continue # 统计每个文件的同步情况 file_success_count = 0 file_error_count = 0 file_total_records = 0 for acc_table, data in table_tasks.items(): record_ids = list(data['record_ids']) log_ids = data['log_ids'] # 读取目标配置 target_conf = tables_map[acc_table] target_schema = target_conf['target_schema'] target_table = target_conf['target_table'] pk_col = target_conf['pk_col'] target_full_name = db_utils.fmt_table(target_schema, target_table) log_processing(f"正在同步表 [{acc_table}] → [{target_table}] ({len(record_ids)} 条记录)") try: # --- A. Access 查新数据 --- ids_placeholders = ','.join(['?'] * len(record_ids)) acc_sql = f"SELECT * FROM [{acc_table}] WHERE [{pk_col}] IN ({ids_placeholders})" acc_cursor.execute(acc_sql, record_ids) new_rows = acc_cursor.fetchall() acc_cols = [col[0] for col in acc_cursor.description] # --- B. SQL Server 删旧插新 (事务) --- # 1. 删除 del_sql = f"DELETE FROM {target_full_name} WHERE [{pk_col}] IN ({ids_placeholders})" sql_cursor.execute(del_sql, record_ids) # 2. 插入 if new_rows: insert_sql = db_utils.generate_insert_sql(target_schema, target_table, acc_cols) try: sql_cursor.execute(f"SET IDENTITY_INSERT {target_full_name} ON") except: pass sql_cursor.executemany(insert_sql, new_rows) try: sql_cursor.execute(f"SET IDENTITY_INSERT {target_full_name} OFF") except: pass # 3. 标记日志 Synced = 1 log_placeholders = ','.join(['?'] * len(log_ids)) update_log_sql = f""" UPDATE {log_full_name} SET {cols['col_synced']} = 1 WHERE {cols['col_log_id']} IN ({log_placeholders}) """ sql_cursor.execute(update_log_sql, log_ids) sql_conn.commit() log_success(f"表 [{target_table}] 同步完成: {len(record_ids)} 条记录") file_success_count += 1 file_total_records += len(record_ids) except Exception as tbl_err: log_error(f"表 [{acc_table}] 同步失败: {tbl_err}") sql_conn.rollback() file_error_count += 1 acc_conn.close() # 关闭 Access 连接 # 输出文件级别的汇总 if file_success_count > 0 or file_error_count > 0: summary = f"文件 [{os.path.basename(clean_path)}] 同步汇总: " summary += f"成功 {file_success_count} 张表 ({file_total_records} 条记录)" if file_error_count > 0: summary += f" | 失败 {file_error_count} 张表" log_sync(summary) return work_done except Exception as e: log_critical(f"全局异常: {e}") return False finally: try: sql_conn.close() except: pass if __name__ == "__main__": log_start("增量同步服务已启动 (配置驱动模式)") log_info(f"轮询间隔: {config.POLL_INTERVAL} 秒") log_info(f"监控配置: {len(config.SYNC_MAPPING)} 个文件") logger.info("=" * 70) while True: try: has_work = process_sync_task() # 如果有工作,说明可能还有积压,休息短一点(0.1s) # 如果没工作,休息标准间隔(5s) time.sleep(0.1 if has_work else config.POLL_INTERVAL) except KeyboardInterrupt: logger.info("=" * 70) log_stop("收到停止信号,服务正在关闭...") break except Exception as e: log_critical(f"主循环崩溃: {e}") time.sleep(5)