Refactor configuration management and remove deprecated files
- Consolidated database, file source, and field mapping configurations into dedicated modules under the `config` directory. - Removed hardcoded database connection details from `migration.py` and replaced them with imports from the new configuration structure. - Updated `ntfy_utils.py` and `run_incremental_sync.py` to utilize the new configuration imports for cleaner code and better maintainability. - Deleted `update_config.py` as its contents have been integrated into the new configuration files. - Added a new `settings.local.json` for managing permissions related to script execution. - Enhanced the structure of the migration tasks and Excel configurations for better organization and clarity.
This commit is contained in:
@@ -13,7 +13,7 @@ from sqlalchemy.engine import URL
|
||||
from sqlalchemy.types import NVARCHAR, Integer, Date
|
||||
|
||||
# 导入配置
|
||||
import update_config as config
|
||||
from config import DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA
|
||||
|
||||
# ================= 抑制 openpyxl 的数据验证警告 =================
|
||||
warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl')
|
||||
@@ -37,19 +37,19 @@ class DataSynchronizer:
|
||||
def __init__(self, force_sync=False):
|
||||
self.force_sync = force_sync
|
||||
self.engine = self._get_db_connection()
|
||||
self.cache_dir = config.CACHE_DIR
|
||||
self.cache_dir = CACHE_DIR
|
||||
|
||||
if not os.path.exists(self.cache_dir):
|
||||
os.makedirs(self.cache_dir)
|
||||
|
||||
def _get_db_connection(self):
|
||||
connection_string = (
|
||||
f"DRIVER={{{config.DB_CONFIG['driver']}}};"
|
||||
f"SERVER={config.DB_CONFIG['server']};"
|
||||
f"DATABASE={config.DB_CONFIG['database']};"
|
||||
f"UID={config.DB_CONFIG['username']};"
|
||||
f"PWD={config.DB_CONFIG['password']};"
|
||||
f"TrustServerCertificate={config.DB_CONFIG.get('TrustServerCertificate', 'no')};"
|
||||
f"DRIVER={{{DB_CONFIG['driver']}}};"
|
||||
f"SERVER={DB_CONFIG['server']};"
|
||||
f"DATABASE={DB_CONFIG['database']};"
|
||||
f"UID={DB_CONFIG['username']};"
|
||||
f"PWD={DB_CONFIG['password']};"
|
||||
f"TrustServerCertificate={DB_CONFIG.get('TrustServerCertificate', 'no')};"
|
||||
)
|
||||
connection_url = URL.create("mssql+pyodbc", query={"odbc_connect": connection_string})
|
||||
return create_engine(connection_url, fast_executemany=True)
|
||||
@@ -94,7 +94,7 @@ class DataSynchronizer:
|
||||
df = df.drop_duplicates(subset=['总排号'], keep='first')
|
||||
|
||||
# 3. 补全列
|
||||
for col in config.TABLE_SCHEMA.keys():
|
||||
for col in TABLE_SCHEMA.keys():
|
||||
if col not in df.columns:
|
||||
df[col] = None
|
||||
|
||||
@@ -102,7 +102,7 @@ class DataSynchronizer:
|
||||
dtype_dict = {}
|
||||
|
||||
# 4. 字段清洗
|
||||
for col, rules in config.TABLE_SCHEMA.items():
|
||||
for col, rules in TABLE_SCHEMA.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
|
||||
@@ -136,7 +136,7 @@ class DataSynchronizer:
|
||||
|
||||
dtype_dict[col] = NVARCHAR(max_len)
|
||||
|
||||
final_cols = list(config.TABLE_SCHEMA.keys())
|
||||
final_cols = list(TABLE_SCHEMA.keys())
|
||||
|
||||
return df[final_cols], dtype_dict
|
||||
|
||||
@@ -162,7 +162,7 @@ class DataSynchronizer:
|
||||
if not df_insert.empty:
|
||||
logger.info("正在执行批量插入...")
|
||||
df_insert.to_sql('executionCardData', self.engine, schema='warehouseOutbound',
|
||||
if_exists='append', index=False, chunksize=config.BATCH_SIZE,
|
||||
if_exists='append', index=False, chunksize=BATCH_SIZE,
|
||||
dtype=dtype_dict)
|
||||
logger.info("批量插入完成。")
|
||||
|
||||
@@ -198,7 +198,7 @@ class DataSynchronizer:
|
||||
logger.info(f"批量更新完成,共影响 {affected_rows} 行。")
|
||||
|
||||
def process_excel_files(self):
|
||||
for cfg in config.EXCEL_CONFIGS:
|
||||
for cfg in EXCEL_CONFIGS:
|
||||
remote_path = cfg['file_path']
|
||||
filename = os.path.basename(remote_path)
|
||||
local_path = os.path.join(self.cache_dir, filename)
|
||||
|
||||
Reference in New Issue
Block a user