Refactor configuration management and remove deprecated files
- Consolidated database, file source, and field mapping configurations into dedicated modules under the `config` directory. - Removed hardcoded database connection details from `migration.py` and replaced them with imports from the new configuration structure. - Updated `ntfy_utils.py` and `run_incremental_sync.py` to utilize the new configuration imports for cleaner code and better maintainability. - Deleted `update_config.py` as its contents have been integrated into the new configuration files. - Added a new `settings.local.json` for managing permissions related to script execution. - Enhanced the structure of the migration tasks and Excel configurations for better organization and clarity.
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import pyodbc
|
||||
import config
|
||||
from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, BATCH_SIZE
|
||||
import db_utils
|
||||
import time
|
||||
import os
|
||||
@@ -103,8 +103,8 @@ def run_full_sync():
|
||||
|
||||
logger.info("=" * 70)
|
||||
log_start(logger, "全量初始化同步")
|
||||
log_info(logger, f"配置文件数: {len(config.SYNC_MAPPING)} 个")
|
||||
log_info(logger, f"批次大小: {config.BATCH_SIZE} 行")
|
||||
log_info(logger, f"配置文件数: {len(SYNC_MAPPING)} 个")
|
||||
log_info(logger, f"批次大小: {BATCH_SIZE} 行")
|
||||
logger.info("=" * 70)
|
||||
|
||||
sync_start_time = time.time()
|
||||
@@ -123,7 +123,7 @@ def run_full_sync():
|
||||
return
|
||||
|
||||
# 第一层循环:遍历配置文件中的所有文件路径
|
||||
for acc_path, tables_map in config.SYNC_MAPPING.items():
|
||||
for acc_path, tables_map in SYNC_MAPPING.items():
|
||||
if not os.path.exists(acc_path):
|
||||
log_skip(logger, f"文件不存在: {acc_path}")
|
||||
continue
|
||||
@@ -184,7 +184,7 @@ def run_full_sync():
|
||||
last_log_time = start_time
|
||||
|
||||
while True:
|
||||
rows = acc_cursor.fetchmany(config.BATCH_SIZE)
|
||||
rows = acc_cursor.fetchmany(BATCH_SIZE)
|
||||
if not rows: break
|
||||
|
||||
sql_cursor.executemany(insert_sql, rows)
|
||||
|
||||
Reference in New Issue
Block a user