Refactor configuration management and remove deprecated files

- Consolidated database, file source, and field mapping configurations into dedicated modules under the `config` directory.
- Removed hardcoded database connection details from `migration.py` and replaced them with imports from the new configuration structure.
- Updated `ntfy_utils.py` and `run_incremental_sync.py` to utilize the new configuration imports for cleaner code and better maintainability.
- Deleted `update_config.py` as its contents have been integrated into the new configuration files.
- Added a new `settings.local.json` for managing permissions related to script execution.
- Enhanced the structure of the migration tasks and Excel configurations for better organization and clarity.
This commit is contained in:
Misaka_Company
2026-01-12 12:49:51 +08:00
parent ee14de0435
commit 83c90f0161
13 changed files with 501 additions and 365 deletions

View File

@@ -13,7 +13,7 @@ from sqlalchemy.engine import URL
from sqlalchemy.types import NVARCHAR, Integer, Date
# 导入配置
import update_config as config
from config import DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA
# ================= 抑制 openpyxl 的数据验证警告 =================
warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl')
@@ -37,19 +37,19 @@ class DataSynchronizer:
def __init__(self, force_sync=False):
self.force_sync = force_sync
self.engine = self._get_db_connection()
self.cache_dir = config.CACHE_DIR
self.cache_dir = CACHE_DIR
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
def _get_db_connection(self):
connection_string = (
f"DRIVER={{{config.DB_CONFIG['driver']}}};"
f"SERVER={config.DB_CONFIG['server']};"
f"DATABASE={config.DB_CONFIG['database']};"
f"UID={config.DB_CONFIG['username']};"
f"PWD={config.DB_CONFIG['password']};"
f"TrustServerCertificate={config.DB_CONFIG.get('TrustServerCertificate', 'no')};"
f"DRIVER={{{DB_CONFIG['driver']}}};"
f"SERVER={DB_CONFIG['server']};"
f"DATABASE={DB_CONFIG['database']};"
f"UID={DB_CONFIG['username']};"
f"PWD={DB_CONFIG['password']};"
f"TrustServerCertificate={DB_CONFIG.get('TrustServerCertificate', 'no')};"
)
connection_url = URL.create("mssql+pyodbc", query={"odbc_connect": connection_string})
return create_engine(connection_url, fast_executemany=True)
@@ -94,7 +94,7 @@ class DataSynchronizer:
df = df.drop_duplicates(subset=['总排号'], keep='first')
# 3. 补全列
for col in config.TABLE_SCHEMA.keys():
for col in TABLE_SCHEMA.keys():
if col not in df.columns:
df[col] = None
@@ -102,7 +102,7 @@ class DataSynchronizer:
dtype_dict = {}
# 4. 字段清洗
for col, rules in config.TABLE_SCHEMA.items():
for col, rules in TABLE_SCHEMA.items():
if col not in df.columns:
continue
@@ -136,7 +136,7 @@ class DataSynchronizer:
dtype_dict[col] = NVARCHAR(max_len)
final_cols = list(config.TABLE_SCHEMA.keys())
final_cols = list(TABLE_SCHEMA.keys())
return df[final_cols], dtype_dict
@@ -162,7 +162,7 @@ class DataSynchronizer:
if not df_insert.empty:
logger.info("正在执行批量插入...")
df_insert.to_sql('executionCardData', self.engine, schema='warehouseOutbound',
if_exists='append', index=False, chunksize=config.BATCH_SIZE,
if_exists='append', index=False, chunksize=BATCH_SIZE,
dtype=dtype_dict)
logger.info("批量插入完成。")
@@ -198,7 +198,7 @@ class DataSynchronizer:
logger.info(f"批量更新完成,共影响 {affected_rows} 行。")
def process_excel_files(self):
for cfg in config.EXCEL_CONFIGS:
for cfg in EXCEL_CONFIGS:
remote_path = cfg['file_path']
filename = os.path.basename(remote_path)
local_path = os.path.join(self.cache_dir, filename)