Refactor configuration management and remove deprecated files
- Consolidated database, file source, and field mapping configurations into dedicated modules under the `config` directory. - Removed hardcoded database connection details from `migration.py` and replaced them with imports from the new configuration structure. - Updated `ntfy_utils.py` and `run_incremental_sync.py` to utilize the new configuration imports for cleaner code and better maintainability. - Deleted `update_config.py` as its contents have been integrated into the new configuration files. - Added a new `settings.local.json` for managing permissions related to script execution. - Enhanced the structure of the migration tasks and Excel configurations for better organization and clarity.
This commit is contained in:
@@ -2,7 +2,7 @@ import time
|
||||
import os
|
||||
import sys
|
||||
import pyodbc
|
||||
import config
|
||||
from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, POLL_INTERVAL, LOG_TABLE_CONFIG
|
||||
import db_utils
|
||||
import logging
|
||||
from logging.handlers import TimedRotatingFileHandler
|
||||
@@ -95,7 +95,7 @@ def log_sync(message):
|
||||
def process_sync_task():
|
||||
"""
|
||||
逻辑重构:
|
||||
遍历 config.SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。
|
||||
遍历 SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。
|
||||
"""
|
||||
sql_conn = db_utils.get_sql_conn()
|
||||
sql_cursor = sql_conn.cursor()
|
||||
@@ -104,12 +104,12 @@ def process_sync_task():
|
||||
# 标记是否有工作被处理(用于控制轮询休眠时间)
|
||||
work_done = False
|
||||
|
||||
cols = config.LOG_TABLE_CONFIG
|
||||
cols = LOG_TABLE_CONFIG
|
||||
log_full_name = db_utils.fmt_table(cols['schema'], cols['table_name'])
|
||||
|
||||
try:
|
||||
# === 核心循环:以配置文件为驱动 ===
|
||||
for clean_path, tables_map in config.SYNC_MAPPING.items():
|
||||
for clean_path, tables_map in SYNC_MAPPING.items():
|
||||
|
||||
# 1. 准备查询条件
|
||||
# 获取该文件下所有需要同步的表名列表
|
||||
@@ -265,8 +265,8 @@ def process_sync_task():
|
||||
|
||||
if __name__ == "__main__":
|
||||
log_start("增量同步服务已启动 (配置驱动模式)")
|
||||
log_info(f"轮询间隔: {config.POLL_INTERVAL} 秒")
|
||||
log_info(f"监控配置: {len(config.SYNC_MAPPING)} 个文件")
|
||||
log_info(f"轮询间隔: {POLL_INTERVAL} 秒")
|
||||
log_info(f"监控配置: {len(SYNC_MAPPING)} 个文件")
|
||||
logger.info("=" * 70)
|
||||
|
||||
while True:
|
||||
@@ -274,7 +274,7 @@ if __name__ == "__main__":
|
||||
has_work = process_sync_task()
|
||||
# 如果有工作,说明可能还有积压,休息短一点(0.1s)
|
||||
# 如果没工作,休息标准间隔(5s)
|
||||
time.sleep(0.1 if has_work else config.POLL_INTERVAL)
|
||||
time.sleep(0.1 if has_work else POLL_INTERVAL)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("=" * 70)
|
||||
log_stop("收到停止信号,服务正在关闭...")
|
||||
|
||||
Reference in New Issue
Block a user