feat: add scheduled sync daemon and TRUNCATE migration mode
- Add sync.py for LAN file sync with atomic writes and mtime-based change detection - Add watch.py daemon that periodically syncs files and migrates only changed tables - Add TRUNCATE mode to migrate.py (--truncate flag) to preserve table structure - Update config.yaml schema with check_interval_minutes and lan_sources - Update README with new features documentation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
186
watch.py
Normal file
186
watch.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""
|
||||
Watch daemon - periodically syncs LAN Excel files and migrates changed tables to SQL Server.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import yaml
|
||||
|
||||
from migrate import load_config, get_connection, create_schema, migrate_table
|
||||
from sync import sync_all_files, determine_tables_to_migrate
|
||||
|
||||
|
||||
def setup_logging(log_file):
|
||||
"""Configure logging to both console and file."""
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
fmt = logging.Formatter(
|
||||
'%(asctime)s [%(levelname)s] %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
|
||||
# Console handler
|
||||
ch = logging.StreamHandler(sys.stdout)
|
||||
ch.setLevel(logging.INFO)
|
||||
ch.setFormatter(fmt)
|
||||
logger.addHandler(ch)
|
||||
|
||||
# File handler
|
||||
fh = logging.FileHandler(log_file, encoding='utf-8')
|
||||
fh.setLevel(logging.INFO)
|
||||
fh.setFormatter(fmt)
|
||||
logger.addHandler(fh)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def run_once(cfg, logger):
|
||||
"""Execute one sync + migrate cycle."""
|
||||
excel_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Excel')
|
||||
|
||||
# 1. Sync files from LAN
|
||||
logger.info("开始同步 LAN 文件...")
|
||||
sync_results = sync_all_files(cfg, excel_dir)
|
||||
|
||||
for rel_path, (status, error_msg) in sync_results.items():
|
||||
if status == 'error':
|
||||
logger.error(f" 同步失败: {rel_path} - {error_msg}")
|
||||
elif status in ('copied', 'updated'):
|
||||
logger.info(f" {status}: {rel_path}")
|
||||
else:
|
||||
logger.debug(f" 跳过: {rel_path}")
|
||||
|
||||
# 2. Determine which tables need migration
|
||||
tables_to_migrate = determine_tables_to_migrate(cfg, sync_results)
|
||||
|
||||
if not tables_to_migrate:
|
||||
logger.info("无文件变更，跳过迁移")
|
||||
return
|
||||
|
||||
logger.info(f"需要迁移的表: {', '.join(sorted(tables_to_migrate))}")
|
||||
|
||||
# 3. Open DB connection and migrate each changed table
|
||||
conn = None
|
||||
try:
|
||||
conn = get_connection(cfg)
|
||||
cursor = conn.cursor()
|
||||
create_schema(cursor, cfg['schema'])
|
||||
|
||||
for table_cfg in cfg['tables']:
|
||||
table_name = table_cfg['table']
|
||||
if table_name not in tables_to_migrate:
|
||||
continue
|
||||
|
||||
try:
|
||||
rows = migrate_table(cursor, table_cfg, cfg, use_truncate=True)
|
||||
logger.info(f" 迁移完成: {table_name} ({rows:,} 行)")
|
||||
except Exception as e:
|
||||
logger.error(f" 迁移失败: {table_name} - {e}", exc_info=True)
|
||||
# Rollback any uncommitted transaction for this table
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cursor.close()
|
||||
except Exception as e:
|
||||
logger.error(f"数据库连接失败: {e}", exc_info=True)
|
||||
finally:
|
||||
if conn:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Excel to SQL Server Watch Daemon')
|
||||
parser.add_argument('--config', default=None, help='配置文件路径 (默认: config.yaml)')
|
||||
parser.add_argument('--once', action='store_true', help='执行一次后退出')
|
||||
parser.add_argument('--interval', type=int, default=None,
|
||||
help='覆盖配置文件中的 check_interval_minutes')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Setup logging
|
||||
log_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'watch.log')
|
||||
logger = setup_logging(log_file)
|
||||
|
||||
# Load config
|
||||
cfg = load_config(args.config)
|
||||
interval = args.interval or cfg.get('check_interval_minutes', 30)
|
||||
|
||||
logger.info("=" * 50)
|
||||
logger.info("Watch daemon 启动")
|
||||
logger.info(f" 同步间隔: {interval} 分钟")
|
||||
logger.info("=" * 50)
|
||||
|
||||
first_run = True
|
||||
|
||||
while True:
|
||||
try:
|
||||
if first_run:
|
||||
logger.info("首次运行，强制同步并迁移所有表...")
|
||||
# On first run, force all tables to migrate by marking all files as 'updated'
|
||||
excel_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Excel')
|
||||
sync_results = sync_all_files(cfg, excel_dir)
|
||||
|
||||
# Log sync results
|
||||
for rel_path, (status, error_msg) in sync_results.items():
|
||||
if status == 'error':
|
||||
logger.error(f" 同步失败: {rel_path} - {error_msg}")
|
||||
else:
|
||||
logger.info(f" {status}: {rel_path}")
|
||||
|
||||
# Force all tables to migrate on first run
|
||||
all_tables = {t['table'] for t in cfg['tables']}
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn = get_connection(cfg)
|
||||
cursor = conn.cursor()
|
||||
create_schema(cursor, cfg['schema'])
|
||||
|
||||
for table_cfg in cfg['tables']:
|
||||
table_name = table_cfg['table']
|
||||
try:
|
||||
rows = migrate_table(cursor, table_cfg, cfg, use_truncate=True)
|
||||
logger.info(f" 迁移完成: {table_name} ({rows:,} 行)")
|
||||
except Exception as e:
|
||||
logger.error(f" 迁移失败: {table_name} - {e}", exc_info=True)
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cursor.close()
|
||||
except Exception as e:
|
||||
logger.error(f"数据库连接失败: {e}", exc_info=True)
|
||||
finally:
|
||||
if conn:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
first_run = False
|
||||
else:
|
||||
run_once(cfg, logger)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"运行周期异常: {e}", exc_info=True)
|
||||
|
||||
if args.once:
|
||||
logger.info("--once 模式，退出")
|
||||
break
|
||||
|
||||
logger.info(f"下次检查: {interval} 分钟后")
|
||||
time.sleep(interval * 60)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user