AllForCompany/BLD_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor logging system to use unified log_utils module and enhance logging messages across multiple scripts

Browse Source
This commit is contained in:
Misaka_Company
2026-01-12 13:55:28 +08:00
parent 83c90f0161
commit 4721634b81
7 changed files with 227 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
init_full_sync.py
View File

@@ -3,79 +3,11 @@ from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, BATCH_SIZE
import db_utils
import time
import os
import logging
from datetime import datetime
from log_utils import (LoggerManager, log_success, log_error, log_warning, log_info, log_processing,
log_skip, log_start, log_file, log_database, log_sync, log_complete)
def setup_logger():
"""初始化日志配置，每次运行生成独立的日志文件"""
# 创建log目录
log_dir = "log"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# 生成带时间戳的日志文件名
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
log_file = os.path.join(log_dir, f"full_sync_{timestamp}.log")
# 配置日志格式
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
handlers=[
logging.FileHandler(log_file, encoding='utf-8'),
logging.StreamHandler() # 同时输出到控制台
]
)
logger = logging.getLogger(__name__)
logger.info(f"📝 日志文件: {log_file}")
return logger
# ================= 日志辅助函数 =================
def log_success(logger, message):
"""成功消息"""
logger.info(f"✅ [成功] {message}")
def log_error(logger, message, exc_info=False):
"""错误消息"""
logger.error(f"❌ [错误] {message}", exc_info=exc_info)
def log_warning(logger, message):
"""警告消息"""
logger.warning(f"⚠️ [警告] {message}")
def log_info(logger, message):
"""信息消息"""
logger.info(f"ℹ️ {message}")
def log_processing(logger, message):
"""处理中消息"""
logger.info(f"🔄 [处理] {message}")
def log_skip(logger, message):
"""跳过消息"""
logger.info(f"⏭️ [跳过] {message}")
def log_start(logger, message):
"""启动消息"""
logger.info(f"🚀 [启动] {message}")
def log_file(logger, message):
"""文件操作消息"""
logger.info(f"📂 [文件] {message}")
def log_database(logger, message):
"""数据库操作消息"""
logger.info(f"💾 [数据库] {message}")
def log_sync(logger, message):
"""同步操作消息"""
logger.info(f"🔃 [同步] {message}")
def log_complete(logger, message):
"""完成消息"""
logger.info(f"🎯 [完成] {message}")
# 初始化日志管理器
LoggerManager("init_full_sync", log_prefix="full_sync")
def format_duration(seconds):
"""格式化时长显示"""
@@ -99,13 +31,11 @@ def has_identity_column(sql_cursor, schema, table):
return sql_cursor.fetchone()[0] > 0
def run_full_sync():
logger = setup_logger()
logger.info("=" * 70)
log_start(logger, "全量初始化同步")
log_info(logger, f"配置文件数: {len(SYNC_MAPPING)} 个")
log_info(logger, f"批次大小: {BATCH_SIZE} 行")
logger.info("=" * 70)
log_info("=" * 70)
log_start( "全量初始化同步")
log_info( f"配置文件数: {len(SYNC_MAPPING)} 个")
log_info( f"批次大小: {BATCH_SIZE} 行")
log_info("=" * 70)
sync_start_time = time.time()
total_tables = 0
@@ -117,18 +47,18 @@ def run_full_sync():
sql_conn = db_utils.get_sql_conn()
sql_cursor = sql_conn.cursor()
sql_cursor.fast_executemany = True
log_database(logger, "SQL Server 连接成功")
log_database( "SQL Server 连接成功")
except Exception as e:
log_error(logger, f"SQL Server 连接失败: {e}")
log_error( f"SQL Server 连接失败: {e}")
return
# 第一层循环:遍历配置文件中的所有文件路径
for acc_path, tables_map in SYNC_MAPPING.items():
if not os.path.exists(acc_path):
log_skip(logger, f"文件不存在: {acc_path}")
log_skip( f"文件不存在: {acc_path}")
continue
log_file(logger, f"开始处理: {os.path.basename(acc_path)}")
log_file( f"开始处理: {os.path.basename(acc_path)}")
file_start_time = time.time()
file_success = 0
file_failed = 0
@@ -137,7 +67,7 @@ def run_full_sync():
try:
acc_conn = db_utils.get_access_conn(acc_path)
acc_cursor = acc_conn.cursor()
log_database(logger, f"Access 文件连接成功: {os.path.basename(acc_path)}")
log_database( f"Access 文件连接成功: {os.path.basename(acc_path)}")
# 第二层循环:遍历该文件下的所有表映射
for acc_table, target_config in tables_map.items():
@@ -146,7 +76,7 @@ def run_full_sync():
target_table = target_config['target_table']
full_target_name = db_utils.fmt_table(target_schema, target_table)
log_processing(logger, f"表 [{acc_table}] → [{target_table}]")
log_processing( f"表 [{acc_table}] → [{target_table}]")
table_start_time = time.time()
try:
@@ -155,12 +85,12 @@ def run_full_sync():
# 2. 获取列结构
columns = [col[0] for col in acc_cursor.description]
log_info(logger, f" 检测到 {len(columns)} 个列")
log_info( f" 检测到 {len(columns)} 个列")
insert_sql = db_utils.generate_insert_sql(target_schema, target_table, columns)
# 3. TRUNCATE 目标表
sql_cursor.execute(f"TRUNCATE TABLE {full_target_name}")
log_info(logger, f" 已清空目标表")
log_info( f" 已清空目标表")
# 4. 检查并启用 IDENTITY_INSERT
has_identity = has_identity_column(sql_cursor, target_schema, target_table)
@@ -170,13 +100,13 @@ def run_full_sync():
try:
sql_cursor.execute(f"SET IDENTITY_INSERT {full_target_name} ON")
identity_enabled = True
log_info(logger, f" 已启用 IDENTITY_INSERT")
log_info( f" 已启用 IDENTITY_INSERT")
except Exception as id_err:
log_error(logger, f" 无法启用 IDENTITY_INSERT: {id_err}")
log_error( f" 无法启用 IDENTITY_INSERT: {id_err}")
raise
# 5. 传输数据
log_info(logger, f" 开始数据传输...")
log_info( f" 开始数据传输...")
acc_cursor.execute(f"SELECT * FROM [{acc_table}]")
table_rows = 0
@@ -195,23 +125,23 @@ def run_full_sync():
if (current_time - last_log_time >= 5) or (table_rows % 10000 == 0):
elapsed = current_time - start_time
rate = table_rows / elapsed if elapsed > 0 else 0
log_info(logger, f" 进度: {table_rows:,} 行 | 速率: {rate:,.0f} 行/秒")
log_info( f" 进度: {table_rows:,} 行 | 速率: {rate:,.0f} 行/秒")
last_log_time = current_time
# 6. 关闭 IDENTITY_INSERT
if identity_enabled:
try:
sql_cursor.execute(f"SET IDENTITY_INSERT {full_target_name} OFF")
log_info(logger, f" 已关闭 IDENTITY_INSERT")
log_info( f" 已关闭 IDENTITY_INSERT")
except Exception as id_err:
log_warning(logger, f" 关闭 IDENTITY_INSERT 时警告: {id_err}")
log_warning( f" 关闭 IDENTITY_INSERT 时警告: {id_err}")
sql_conn.commit()
# 显示最终统计
table_time = time.time() - table_start_time
final_rate = table_rows / table_time if table_time > 0 else 0
log_success(logger, f"表 [{target_table}] 完成: {table_rows:,} 行 | 速率: {final_rate:,.0f} 行/秒 | 用时: {format_duration(table_time)}")
log_success( f"表 [{target_table}] 完成: {table_rows:,} 行 | 速率: {final_rate:,.0f} 行/秒 | 用时: {format_duration(table_time)}")
success_tables += 1
file_success += 1
@@ -221,7 +151,7 @@ def run_full_sync():
except Exception as tbl_err:
failed_tables += 1
file_failed += 1
log_error(logger, f"表 [{acc_table}] 同步失败: {tbl_err}", exc_info=True)
log_error( f"表 [{acc_table}] 同步失败: {tbl_err}", exc_info=True)
sql_conn.rollback()
# 确保清理 IDENTITY_INSERT 状态
@@ -239,24 +169,24 @@ def run_full_sync():
if file_failed > 0:
summary += f" | 失败 {file_failed} 张表"
summary += f" | 共 {file_rows:,} 行 | 用时: {format_duration(file_time)}"
log_sync(logger, summary)
log_sync( summary)
except Exception as file_err:
log_error(logger, f"文件 [{os.path.basename(acc_path)}] 处理失败: {file_err}", exc_info=True)
log_error( f"文件 [{os.path.basename(acc_path)}] 处理失败: {file_err}", exc_info=True)
sql_conn.close()
# 总结统计
sync_time = time.time() - sync_start_time
logger.info("\n" + "=" * 70)
log_complete(logger, "全量同步任务结束")
logger.info("-" * 70)
log_info(logger, f"总表数: {total_tables} 张 | 成功: {success_tables} 张 | 失败: {failed_tables} 张")
log_info(logger, f"总行数: {total_rows:,} 行")
log_info(logger, f"总用时: {format_duration(sync_time)}")
log_info("\n" + "=" * 70)
log_complete( "全量同步任务结束")
log_info("-" * 70)
log_info( f"总表数: {total_tables} 张 | 成功: {success_tables} 张 | 失败: {failed_tables} 张")
log_info( f"总行数: {total_rows:,} 行")
log_info( f"总用时: {format_duration(sync_time)}")
if total_rows > 0 and sync_time > 0:
log_info(logger, f"整体平均速率: {total_rows/sync_time:,.0f} 行/秒")
logger.info("=" * 70)
log_info( f"整体平均速率: {total_rows/sync_time:,.0f} 行/秒")
log_info("=" * 70)
if __name__ == "__main__":
run_full_sync()