feat: unified logging module and remove redundant first_run logic
- Add logger.py with TimedRotatingFileHandler (daily rotation, 30-day retention) - Replace all print() in migrate.py with logging calls - Add logging to sync.py for sync status tracking - Refactor watch.py to use logger.py, remove duplicate first_run branch - All logs now go to logs/app.log consistently - Add logging config section to config.yaml, add logs/ to .gitignore Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
55
migrate.py
55
migrate.py
@@ -5,6 +5,7 @@ Configuration is loaded from config.yaml.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
@@ -14,6 +15,8 @@ import openpyxl
|
||||
import pyodbc
|
||||
import yaml
|
||||
|
||||
logger = logging.getLogger('app')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 配置加载
|
||||
@@ -429,7 +432,7 @@ def migrate_table(cursor, table_config, cfg, dry_run=False, use_truncate=False):
|
||||
primary_key = cfg['primary_key']
|
||||
excel_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Excel')
|
||||
|
||||
print(f"\n 迁移 {table_name} ...")
|
||||
logger.info(f"迁移 {table_name} ...")
|
||||
|
||||
# 1. 读取并合并数据
|
||||
headers, actual_pk, merged = merge_sources(sources, excel_dir, primary_key, cfg)
|
||||
@@ -438,31 +441,31 @@ def migrate_table(cursor, table_config, cfg, dry_run=False, use_truncate=False):
|
||||
for src in sources:
|
||||
fpath = os.path.join(excel_dir, src['file'])
|
||||
_, _, data = read_sheet(fpath, src['sheet'], primary_key, cfg)
|
||||
print(f" 读取: {src['file']} > {src['sheet']} → {len(data):,} 行")
|
||||
logger.info(f" 读取: {src['file']} > {src['sheet']} → {len(data):,} 行")
|
||||
|
||||
print(f" 合并去重后: {len(merged):,} 行, {len(headers)} 列 (主键列: {actual_pk})")
|
||||
logger.info(f" 合并去重后: {len(merged):,} 行, {len(headers)} 列 (主键列: {actual_pk})")
|
||||
|
||||
if dry_run:
|
||||
print(f" [DRY-RUN] 建表 SQL:")
|
||||
logger.info(f" [DRY-RUN] 建表 SQL:")
|
||||
type_rules = cfg['type_rules']
|
||||
col_defs = ['[id] INT IDENTITY(1,1)']
|
||||
for h in headers:
|
||||
sql_type = infer_sql_type(h, type_rules, actual_pk)
|
||||
col_defs.append(f' [{h}] {sql_type}')
|
||||
col_defs.append(f' CONSTRAINT [PK_{table_name}] PRIMARY KEY ([{actual_pk}])')
|
||||
print(f" CREATE TABLE [{cfg['schema']}].[{table_name}] (")
|
||||
print(',\n'.join(col_defs))
|
||||
print(" );")
|
||||
logger.info(f" CREATE TABLE [{cfg['schema']}].[{table_name}] (")
|
||||
logger.info(',\n'.join(col_defs))
|
||||
logger.info(" );")
|
||||
return len(merged)
|
||||
|
||||
# 2. 建表
|
||||
create_sql = ensure_table(cursor, table_name, headers, cfg, actual_pk, use_truncate=use_truncate)
|
||||
action = "TRUNCATE" if use_truncate and "TRUNCATE" in (create_sql or "") else "建表"
|
||||
print(f" {action}: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
|
||||
logger.info(f" {action}: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
|
||||
|
||||
# 3. 插入数据
|
||||
total = batch_insert(cursor, table_name, headers, merged, cfg, actual_pk)
|
||||
print(f" 插入: {total:,} 行 done")
|
||||
logger.info(f" 插入: {total:,} 行 done")
|
||||
|
||||
return total
|
||||
|
||||
@@ -480,15 +483,25 @@ def main():
|
||||
# 加载配置
|
||||
cfg = load_config(args.config)
|
||||
|
||||
# 初始化日志
|
||||
from logger import setup_logger
|
||||
log_cfg = cfg.get('logging', {})
|
||||
setup_logger(
|
||||
'app',
|
||||
log_dir=log_cfg.get('log_dir'),
|
||||
level=log_cfg.get('level', 'INFO'),
|
||||
max_keep_days=log_cfg.get('max_keep_days', 30),
|
||||
)
|
||||
|
||||
# 按参数筛选表
|
||||
if args.table:
|
||||
specified = set(args.table)
|
||||
tables = [t for t in cfg['tables'] if t['table'] in specified]
|
||||
missing = specified - {t['table'] for t in tables}
|
||||
if missing:
|
||||
print(f" 错误: 未找到表 {', '.join(missing)}")
|
||||
logger.error(f"未找到表 {', '.join(missing)}")
|
||||
available = ', '.join(t['table'] for t in cfg['tables'])
|
||||
print(f" 可用表: {available}")
|
||||
logger.info(f"可用表: {available}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
tables = cfg['tables']
|
||||
@@ -496,11 +509,11 @@ def main():
|
||||
total_tables = len(tables)
|
||||
primary_key = cfg['primary_key']
|
||||
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Excel → SQL Server 迁移工具")
|
||||
print(f" 目标: [{cfg['schema']}], {total_tables} 张表")
|
||||
print(f" 模式: {'DRY-RUN' if args.dry_run else 'LIVE'}")
|
||||
print(f"{'=' * 60}")
|
||||
logger.info("=" * 50)
|
||||
logger.info(f"Excel → SQL Server 迁移工具")
|
||||
logger.info(f" 目标: [{cfg['schema']}], {total_tables} 张表")
|
||||
logger.info(f" 模式: {'DRY-RUN' if args.dry_run else 'LIVE'}")
|
||||
logger.info("=" * 50)
|
||||
|
||||
conn = None
|
||||
cursor = None
|
||||
@@ -509,11 +522,11 @@ def main():
|
||||
conn = get_connection(cfg)
|
||||
cursor = conn.cursor()
|
||||
create_schema(cursor, cfg['schema'])
|
||||
print(f"\n Schema [{cfg['schema']}] 已就绪")
|
||||
logger.info(f"Schema [{cfg['schema']}] 已就绪")
|
||||
|
||||
total_rows = 0
|
||||
for i, table_cfg in enumerate(tables, 1):
|
||||
print(f"\n[{i}/{total_tables}]", end='')
|
||||
logger.info(f"[{i}/{total_tables}]")
|
||||
rows = migrate_table(cursor, table_cfg, cfg, dry_run=args.dry_run,
|
||||
use_truncate=args.truncate)
|
||||
total_rows += rows
|
||||
@@ -522,9 +535,9 @@ def main():
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" 完成! 共迁移 {total_tables} 张表, {total_rows:,} 行")
|
||||
print(f"{'=' * 60}")
|
||||
logger.info("=" * 50)
|
||||
logger.info(f"完成! 共迁移 {total_tables} 张表, {total_rows:,} 行")
|
||||
logger.info("=" * 50)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user