Enhance logging and synchronization features across multiple scripts
- Added detailed logging functionality to init_full_sync.py and run_incremental_sync.py for better tracking of synchronization processes. - Updated configuration mappings in config.py to include additional Access database tables. - Improved error handling and user feedback in vbareplace.py, including the ability to refresh linked tables in Access. - Adjusted polling intervals and batch sizes for performance optimization.
This commit is contained in:
Misaka_Company
@@ -1,83 +1,196 @@
|
||||
# init_full_sync.py
|
||||
import pyodbc
|
||||
import config
|
||||
import db_utils
|
||||
import time
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
def setup_logger():
|
||||
"""初始化日志配置，每次运行生成独立的日志文件"""
|
||||
# 创建log目录
|
||||
log_dir = "log"
|
||||
if not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
|
||||
# 生成带时间戳的日志文件名
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
log_file = os.path.join(log_dir, f"full_sync_{timestamp}.log")
|
||||
|
||||
# 配置日志格式
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(levelname)s] %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S',
|
||||
handlers=[
|
||||
logging.FileHandler(log_file, encoding='utf-8'),
|
||||
logging.StreamHandler() # 同时输出到控制台
|
||||
]
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info(f"日志文件: {log_file}")
|
||||
return logger
|
||||
|
||||
def format_duration(seconds):
|
||||
"""格式化时长显示"""
|
||||
if seconds < 60:
|
||||
return f"{seconds:.1f}秒"
|
||||
elif seconds < 3600:
|
||||
return f"{seconds/60:.1f}分钟"
|
||||
else:
|
||||
return f"{seconds/3600:.1f}小时"
|
||||
|
||||
def has_identity_column(sql_cursor, schema, table):
|
||||
"""检查表是否包含标识列"""
|
||||
query = """
|
||||
SELECT COUNT(*)
|
||||
FROM sys.columns c
|
||||
JOIN sys.tables t ON c.object_id = t.object_id
|
||||
JOIN sys.schemas s ON t.schema_id = s.schema_id
|
||||
WHERE s.name = ? AND t.name = ? AND c.is_identity = 1
|
||||
"""
|
||||
sql_cursor.execute(query, (schema, table))
|
||||
return sql_cursor.fetchone()[0] > 0
|
||||
|
||||
def run_full_sync():
|
||||
print("=== 开始全量初始化同步 ===")
|
||||
print("注意：将基于 config.SYNC_MAPPING 中的文件路径进行处理")
|
||||
logger = setup_logger()
|
||||
|
||||
sql_conn = db_utils.get_sql_conn()
|
||||
sql_cursor = sql_conn.cursor()
|
||||
sql_cursor.fast_executemany = True
|
||||
logger.info("=== 开始全量初始化同步 ===")
|
||||
logger.info("注意:将基于 config.SYNC_MAPPING 中的文件路径进行处理")
|
||||
|
||||
sync_start_time = time.time()
|
||||
total_tables = 0
|
||||
success_tables = 0
|
||||
failed_tables = 0
|
||||
total_rows = 0
|
||||
|
||||
try:
|
||||
sql_conn = db_utils.get_sql_conn()
|
||||
sql_cursor = sql_conn.cursor()
|
||||
sql_cursor.fast_executemany = True
|
||||
logger.info("✓ SQL Server 连接成功")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ SQL Server 连接失败: {e}")
|
||||
return
|
||||
|
||||
# 第一层循环：遍历配置文件中的所有文件路径
|
||||
# 第一层循环:遍历配置文件中的所有文件路径
|
||||
for acc_path, tables_map in config.SYNC_MAPPING.items():
|
||||
if not os.path.exists(acc_path):
|
||||
print(f"⚠️ [跳过] 文件不存在: {acc_path}")
|
||||
logger.warning(f"⚠️ [跳过] 文件不存在: {acc_path}")
|
||||
continue
|
||||
|
||||
print(f"\n📂 正在处理文件: {acc_path}")
|
||||
logger.info(f"\n📂 正在处理文件: {acc_path}")
|
||||
|
||||
try:
|
||||
acc_conn = db_utils.get_access_conn(acc_path)
|
||||
acc_cursor = acc_conn.cursor()
|
||||
logger.info(f" ✓ Access 文件连接成功")
|
||||
|
||||
# 第二层循环：遍历该文件下的所有表映射
|
||||
# 第二层循环:遍历该文件下的所有表映射
|
||||
for acc_table, target_config in tables_map.items():
|
||||
total_tables += 1
|
||||
target_schema = target_config['target_schema']
|
||||
target_table = target_config['target_table']
|
||||
full_target_name = db_utils.fmt_table(target_schema, target_table)
|
||||
|
||||
print(f" 👉 同步表: [{acc_table}] -> {full_target_name}")
|
||||
logger.info(f" 👉 开始同步表: [{acc_table}] -> {full_target_name}")
|
||||
table_start_time = time.time()
|
||||
|
||||
try:
|
||||
# 1. 检查 Access 表是否存在
|
||||
acc_cursor.execute(f"SELECT TOP 1 * FROM [{acc_table}]")
|
||||
logger.info(f" ✓ Access 表 [{acc_table}] 存在")
|
||||
|
||||
# 2. 获取列结构
|
||||
columns = [col[0] for col in acc_cursor.description]
|
||||
logger.info(f" ✓ 获取到 {len(columns)} 个列: {', '.join(columns)}")
|
||||
insert_sql = db_utils.generate_insert_sql(target_schema, target_table, columns)
|
||||
|
||||
# 3. TRUNCATE 目标表
|
||||
sql_cursor.execute(f"TRUNCATE TABLE {full_target_name}")
|
||||
logger.info(f" ✓ 已清空目标表")
|
||||
|
||||
# 4. 开启 IDENTITY_INSERT (尝试)
|
||||
try: sql_cursor.execute(f"SET IDENTITY_INSERT {full_target_name} ON")
|
||||
except: pass
|
||||
# 4. 检查并启用 IDENTITY_INSERT
|
||||
has_identity = has_identity_column(sql_cursor, target_schema, target_table)
|
||||
identity_enabled = False
|
||||
|
||||
if has_identity:
|
||||
try:
|
||||
sql_cursor.execute(f"SET IDENTITY_INSERT {full_target_name} ON")
|
||||
identity_enabled = True
|
||||
logger.info(f" ✓ 已启用 IDENTITY_INSERT")
|
||||
except Exception as id_err:
|
||||
logger.error(f" ⚠️ 无法启用 IDENTITY_INSERT: {id_err}")
|
||||
raise
|
||||
|
||||
# 5. 传输数据
|
||||
print(f" 正在读取并写入数据...")
|
||||
logger.info(f" 正在读取并写入数据...")
|
||||
acc_cursor.execute(f"SELECT * FROM [{acc_table}]")
|
||||
|
||||
total_rows = 0
|
||||
table_rows = 0
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
rows = acc_cursor.fetchmany(config.BATCH_SIZE)
|
||||
if not rows: break
|
||||
|
||||
sql_cursor.executemany(insert_sql, rows)
|
||||
total_rows += len(rows)
|
||||
print(f" 已写入 {total_rows} 行...", end='\r')
|
||||
table_rows += len(rows)
|
||||
|
||||
# 定期记录进度到日志
|
||||
if table_rows % (config.BATCH_SIZE * 10) == 0:
|
||||
elapsed = time.time() - start_time
|
||||
rate = table_rows / elapsed if elapsed > 0 else 0
|
||||
logger.info(f" 进度: 已写入 {table_rows:,} 行 | 速率: {rate:,.0f} 行/秒")
|
||||
|
||||
# 6. 关闭 IDENTITY_INSERT
|
||||
try: sql_cursor.execute(f"SET IDENTITY_INSERT {full_target_name} OFF")
|
||||
except: pass
|
||||
if identity_enabled:
|
||||
try:
|
||||
sql_cursor.execute(f"SET IDENTITY_INSERT {full_target_name} OFF")
|
||||
logger.info(f" ✓ 已关闭 IDENTITY_INSERT")
|
||||
except Exception as id_err:
|
||||
logger.warning(f" ⚠️ 关闭 IDENTITY_INSERT 时警告: {id_err}")
|
||||
|
||||
sql_conn.commit()
|
||||
print(f"\n ✅ 完成。")
|
||||
|
||||
# 显示最终统计
|
||||
table_time = time.time() - table_start_time
|
||||
final_rate = table_rows / table_time if table_time > 0 else 0
|
||||
logger.info(f" ✅ 完成。共 {table_rows:,} 行 | 平均速率: {final_rate:,.0f} 行/秒 | 总用时: {format_duration(table_time)}")
|
||||
|
||||
success_tables += 1
|
||||
total_rows += table_rows
|
||||
|
||||
except Exception as tbl_err:
|
||||
print(f" ❌ 表级错误: {tbl_err}")
|
||||
failed_tables += 1
|
||||
logger.error(f" ❌ 表级错误: {tbl_err}", exc_info=True)
|
||||
sql_conn.rollback()
|
||||
|
||||
# 确保清理 IDENTITY_INSERT 状态
|
||||
try:
|
||||
sql_cursor.execute(f"SET IDENTITY_INSERT {full_target_name} OFF")
|
||||
except:
|
||||
pass
|
||||
|
||||
acc_conn.close()
|
||||
logger.info(f" ✓ Access 文件处理完成")
|
||||
|
||||
except Exception as file_err:
|
||||
print(f" ❌ 文件级错误: {file_err}")
|
||||
logger.error(f" ❌ 文件级错误: {file_err}", exc_info=True)
|
||||
|
||||
sql_conn.close()
|
||||
print("\n=== 全量同步结束 ===")
|
||||
|
||||
# 总结统计
|
||||
sync_time = time.time() - sync_start_time
|
||||
logger.info("\n" + "="*60)
|
||||
logger.info("=== 全量同步结束 ===")
|
||||
logger.info(f"总表数: {total_tables} | 成功: {success_tables} | 失败: {failed_tables}")
|
||||
logger.info(f"总行数: {total_rows:,} 行")
|
||||
logger.info(f"总用时: {format_duration(sync_time)}")
|
||||
if total_rows > 0 and sync_time > 0:
|
||||
logger.info(f"整体平均速率: {total_rows/sync_time:,.0f} 行/秒")
|
||||
logger.info("="*60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_full_sync()
|
||||
Reference in New Issue
Block a user