feat: unified logging module and remove redundant first_run logic

- Add logger.py with TimedRotatingFileHandler (daily rotation, 30-day retention)
- Replace all print() in migrate.py with logging calls
- Add logging to sync.py for sync status tracking
- Refactor watch.py to use logger.py, remove duplicate first_run branch
- All logs now go to logs/app.log consistently
- Add logging config section to config.yaml, add logs/ to .gitignore

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-06-01 13:56:23 +08:00
parent 7edbd987f0
commit d224b98893
5 changed files with 124 additions and 113 deletions

View File

@@ -5,6 +5,7 @@ Configuration is loaded from config.yaml.
"""
import argparse
import logging
import os
import sys
from collections import OrderedDict
@@ -14,6 +15,8 @@ import openpyxl
import pyodbc
import yaml
logger = logging.getLogger('app')
# ============================================================
# 配置加载
@@ -429,7 +432,7 @@ def migrate_table(cursor, table_config, cfg, dry_run=False, use_truncate=False):
primary_key = cfg['primary_key']
excel_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Excel')
print(f"\n 迁移 {table_name} ...")
logger.info(f"迁移 {table_name} ...")
# 1. 读取并合并数据
headers, actual_pk, merged = merge_sources(sources, excel_dir, primary_key, cfg)
@@ -438,31 +441,31 @@ def migrate_table(cursor, table_config, cfg, dry_run=False, use_truncate=False):
for src in sources:
fpath = os.path.join(excel_dir, src['file'])
_, _, data = read_sheet(fpath, src['sheet'], primary_key, cfg)
print(f" 读取: {src['file']} > {src['sheet']}{len(data):,}")
logger.info(f" 读取: {src['file']} > {src['sheet']}{len(data):,}")
print(f" 合并去重后: {len(merged):,} 行, {len(headers)} 列 (主键列: {actual_pk})")
logger.info(f" 合并去重后: {len(merged):,} 行, {len(headers)} 列 (主键列: {actual_pk})")
if dry_run:
print(f" [DRY-RUN] 建表 SQL:")
logger.info(f" [DRY-RUN] 建表 SQL:")
type_rules = cfg['type_rules']
col_defs = ['[id] INT IDENTITY(1,1)']
for h in headers:
sql_type = infer_sql_type(h, type_rules, actual_pk)
col_defs.append(f' [{h}] {sql_type}')
col_defs.append(f' CONSTRAINT [PK_{table_name}] PRIMARY KEY ([{actual_pk}])')
print(f" CREATE TABLE [{cfg['schema']}].[{table_name}] (")
print(',\n'.join(col_defs))
print(" );")
logger.info(f" CREATE TABLE [{cfg['schema']}].[{table_name}] (")
logger.info(',\n'.join(col_defs))
logger.info(" );")
return len(merged)
# 2. 建表
create_sql = ensure_table(cursor, table_name, headers, cfg, actual_pk, use_truncate=use_truncate)
action = "TRUNCATE" if use_truncate and "TRUNCATE" in (create_sql or "") else "建表"
print(f" {action}: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
logger.info(f" {action}: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
# 3. 插入数据
total = batch_insert(cursor, table_name, headers, merged, cfg, actual_pk)
print(f" 插入: {total:,} 行 done")
logger.info(f" 插入: {total:,} 行 done")
return total
@@ -480,15 +483,25 @@ def main():
# 加载配置
cfg = load_config(args.config)
# 初始化日志
from logger import setup_logger
log_cfg = cfg.get('logging', {})
setup_logger(
'app',
log_dir=log_cfg.get('log_dir'),
level=log_cfg.get('level', 'INFO'),
max_keep_days=log_cfg.get('max_keep_days', 30),
)
# 按参数筛选表
if args.table:
specified = set(args.table)
tables = [t for t in cfg['tables'] if t['table'] in specified]
missing = specified - {t['table'] for t in tables}
if missing:
print(f" 错误: 未找到表 {', '.join(missing)}")
logger.error(f"未找到表 {', '.join(missing)}")
available = ', '.join(t['table'] for t in cfg['tables'])
print(f" 可用表: {available}")
logger.info(f"可用表: {available}")
sys.exit(1)
else:
tables = cfg['tables']
@@ -496,11 +509,11 @@ def main():
total_tables = len(tables)
primary_key = cfg['primary_key']
print(f"{'=' * 60}")
print(f" Excel → SQL Server 迁移工具")
print(f" 目标: [{cfg['schema']}], {total_tables} 张表")
print(f" 模式: {'DRY-RUN' if args.dry_run else 'LIVE'}")
print(f"{'=' * 60}")
logger.info("=" * 50)
logger.info(f"Excel → SQL Server 迁移工具")
logger.info(f" 目标: [{cfg['schema']}], {total_tables} 张表")
logger.info(f" 模式: {'DRY-RUN' if args.dry_run else 'LIVE'}")
logger.info("=" * 50)
conn = None
cursor = None
@@ -509,11 +522,11 @@ def main():
conn = get_connection(cfg)
cursor = conn.cursor()
create_schema(cursor, cfg['schema'])
print(f"\n Schema [{cfg['schema']}] 已就绪")
logger.info(f"Schema [{cfg['schema']}] 已就绪")
total_rows = 0
for i, table_cfg in enumerate(tables, 1):
print(f"\n[{i}/{total_tables}]", end='')
logger.info(f"[{i}/{total_tables}]")
rows = migrate_table(cursor, table_cfg, cfg, dry_run=args.dry_run,
use_truncate=args.truncate)
total_rows += rows
@@ -522,9 +535,9 @@ def main():
cursor.close()
conn.close()
print(f"\n{'=' * 60}")
print(f" 完成! 共迁移 {total_tables} 张表, {total_rows:,}")
print(f"{'=' * 60}")
logger.info("=" * 50)
logger.info(f"完成! 共迁移 {total_tables} 张表, {total_rows:,}")
logger.info("=" * 50)
if __name__ == '__main__':