feat: add scheduled sync daemon and TRUNCATE migration mode
- Add sync.py for LAN file sync with atomic writes and mtime-based change detection - Add watch.py daemon that periodically syncs files and migrates only changed tables - Add TRUNCATE mode to migrate.py (--truncate flag) to preserve table structure - Update config.yaml schema with check_interval_minutes and lan_sources - Update README with new features documentation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
68
migrate.py
68
migrate.py
@@ -216,6 +216,62 @@ def infer_sql_type(col_name, type_rules, primary_key):
|
||||
return type_rules.get('default_type', 'NVARCHAR(500)')
|
||||
|
||||
|
||||
def table_exists(cursor, schema, table_name):
|
||||
"""检查表是否已存在"""
|
||||
full_name = f'[{schema}].[{table_name}]'
|
||||
cursor.execute(f"SELECT OBJECT_ID('{full_name}', 'U')")
|
||||
row = cursor.fetchone()
|
||||
return row[0] is not None
|
||||
|
||||
|
||||
def get_existing_columns(cursor, schema, table_name):
|
||||
"""查询现有表的列名集合(排除 id 自增列)"""
|
||||
full_name = f'[{schema}].[{table_name}]'
|
||||
cursor.execute(f"""
|
||||
SELECT COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
|
||||
ORDER BY ORDINAL_POSITION
|
||||
""", (schema, table_name))
|
||||
cols = {row[0] for row in cursor.fetchall()}
|
||||
cols.discard('id')
|
||||
return cols
|
||||
|
||||
|
||||
def ensure_table(cursor, table_name, headers, cfg, actual_pk, use_truncate=False):
|
||||
"""确保目标表就绪。
|
||||
- use_truncate=False(默认)→ 调用 create_table(),DROP + CREATE
|
||||
- use_truncate=True →
|
||||
1. 表不存在 → CREATE TABLE
|
||||
2. 表存在且列名集合一致 → TRUNCATE TABLE
|
||||
3. 表存在但列名有变化 → DROP + CREATE(兜底)
|
||||
"""
|
||||
if not use_truncate:
|
||||
return create_table(cursor, table_name, headers, cfg, actual_pk)
|
||||
|
||||
schema = cfg['schema']
|
||||
full_name = f'[{schema}].[{table_name}]'
|
||||
|
||||
if not table_exists(cursor, schema, table_name):
|
||||
# 表不存在,走正常 CREATE
|
||||
return create_table(cursor, table_name, headers, cfg, actual_pk)
|
||||
|
||||
# 比较列名集合
|
||||
existing_cols = get_existing_columns(cursor, schema, table_name)
|
||||
new_cols = set(headers)
|
||||
# 主键列也应包含在比较中
|
||||
new_cols.add(actual_pk)
|
||||
|
||||
if existing_cols == new_cols:
|
||||
# 列名一致,TRUNCATE
|
||||
cursor.execute(f"TRUNCATE TABLE {full_name}")
|
||||
cursor.commit()
|
||||
return f"TRUNCATE TABLE {full_name}"
|
||||
else:
|
||||
# 列名有变化,兜底 DROP + CREATE
|
||||
return create_table(cursor, table_name, headers, cfg, actual_pk)
|
||||
|
||||
|
||||
def create_table(cursor, table_name, headers, cfg, actual_pk):
|
||||
"""动态创建表。如已存在则先 DROP。"""
|
||||
schema = cfg['schema']
|
||||
@@ -366,7 +422,7 @@ def batch_insert(cursor, table_name, headers, data_dict, cfg, actual_pk):
|
||||
# 主流程
|
||||
# ============================================================
|
||||
|
||||
def migrate_table(cursor, table_config, cfg, dry_run=False):
|
||||
def migrate_table(cursor, table_config, cfg, dry_run=False, use_truncate=False):
|
||||
"""迁移单个表"""
|
||||
table_name = table_config['table']
|
||||
sources = table_config['sources']
|
||||
@@ -400,8 +456,9 @@ def migrate_table(cursor, table_config, cfg, dry_run=False):
|
||||
return len(merged)
|
||||
|
||||
# 2. 建表
|
||||
create_sql = create_table(cursor, table_name, headers, cfg, actual_pk)
|
||||
print(f" 建表: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
|
||||
create_sql = ensure_table(cursor, table_name, headers, cfg, actual_pk, use_truncate=use_truncate)
|
||||
action = "TRUNCATE" if use_truncate and "TRUNCATE" in (create_sql or "") else "建表"
|
||||
print(f" {action}: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
|
||||
|
||||
# 3. 插入数据
|
||||
total = batch_insert(cursor, table_name, headers, merged, cfg, actual_pk)
|
||||
@@ -414,6 +471,8 @@ def main():
|
||||
parser = argparse.ArgumentParser(description='Excel to SQL Server Migration')
|
||||
parser.add_argument('--config', default=None, help='配置文件路径 (默认: config.yaml)')
|
||||
parser.add_argument('--dry-run', action='store_true', help='仅预览,不写入数据库')
|
||||
parser.add_argument('--truncate', action='store_true',
|
||||
help='使用 TRUNCATE 模式(表存在时截断而非重建)')
|
||||
parser.add_argument('--table', nargs='+', default=None,
|
||||
help='只迁移指定的表,例如 --table PG_2022 TM_2025')
|
||||
args = parser.parse_args()
|
||||
@@ -455,7 +514,8 @@ def main():
|
||||
total_rows = 0
|
||||
for i, table_cfg in enumerate(tables, 1):
|
||||
print(f"\n[{i}/{total_tables}]", end='')
|
||||
rows = migrate_table(cursor, table_cfg, cfg, dry_run=args.dry_run)
|
||||
rows = migrate_table(cursor, table_cfg, cfg, dry_run=args.dry_run,
|
||||
use_truncate=args.truncate)
|
||||
total_rows += rows
|
||||
|
||||
if not args.dry_run:
|
||||
|
||||
Reference in New Issue
Block a user