feat: add scheduled sync daemon and TRUNCATE migration mode

- Add sync.py for LAN file sync with atomic writes and mtime-based change detection
- Add watch.py daemon that periodically syncs files and migrates only changed tables
- Add TRUNCATE mode to migrate.py (--truncate flag) to preserve table structure
- Update config.yaml schema with check_interval_minutes and lan_sources
- Update README with new features documentation

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-06-01 13:29:23 +08:00
parent 07ab64bf38
commit 7edbd987f0
5 changed files with 390 additions and 10 deletions

View File

@@ -216,6 +216,62 @@ def infer_sql_type(col_name, type_rules, primary_key):
return type_rules.get('default_type', 'NVARCHAR(500)')
def table_exists(cursor, schema, table_name):
"""检查表是否已存在"""
full_name = f'[{schema}].[{table_name}]'
cursor.execute(f"SELECT OBJECT_ID('{full_name}', 'U')")
row = cursor.fetchone()
return row[0] is not None
def get_existing_columns(cursor, schema, table_name):
"""查询现有表的列名集合(排除 id 自增列)"""
full_name = f'[{schema}].[{table_name}]'
cursor.execute(f"""
SELECT COLUMN_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY ORDINAL_POSITION
""", (schema, table_name))
cols = {row[0] for row in cursor.fetchall()}
cols.discard('id')
return cols
def ensure_table(cursor, table_name, headers, cfg, actual_pk, use_truncate=False):
"""确保目标表就绪。
- use_truncate=False默认→ 调用 create_table()DROP + CREATE
- use_truncate=True →
1. 表不存在 → CREATE TABLE
2. 表存在且列名集合一致 → TRUNCATE TABLE
3. 表存在但列名有变化 → DROP + CREATE兜底
"""
if not use_truncate:
return create_table(cursor, table_name, headers, cfg, actual_pk)
schema = cfg['schema']
full_name = f'[{schema}].[{table_name}]'
if not table_exists(cursor, schema, table_name):
# 表不存在,走正常 CREATE
return create_table(cursor, table_name, headers, cfg, actual_pk)
# 比较列名集合
existing_cols = get_existing_columns(cursor, schema, table_name)
new_cols = set(headers)
# 主键列也应包含在比较中
new_cols.add(actual_pk)
if existing_cols == new_cols:
# 列名一致TRUNCATE
cursor.execute(f"TRUNCATE TABLE {full_name}")
cursor.commit()
return f"TRUNCATE TABLE {full_name}"
else:
# 列名有变化,兜底 DROP + CREATE
return create_table(cursor, table_name, headers, cfg, actual_pk)
def create_table(cursor, table_name, headers, cfg, actual_pk):
"""动态创建表。如已存在则先 DROP。"""
schema = cfg['schema']
@@ -366,7 +422,7 @@ def batch_insert(cursor, table_name, headers, data_dict, cfg, actual_pk):
# 主流程
# ============================================================
def migrate_table(cursor, table_config, cfg, dry_run=False):
def migrate_table(cursor, table_config, cfg, dry_run=False, use_truncate=False):
"""迁移单个表"""
table_name = table_config['table']
sources = table_config['sources']
@@ -400,8 +456,9 @@ def migrate_table(cursor, table_config, cfg, dry_run=False):
return len(merged)
# 2. 建表
create_sql = create_table(cursor, table_name, headers, cfg, actual_pk)
print(f" 建表: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
create_sql = ensure_table(cursor, table_name, headers, cfg, actual_pk, use_truncate=use_truncate)
action = "TRUNCATE" if use_truncate and "TRUNCATE" in (create_sql or "") else "建表"
print(f" {action}: [{cfg['schema']}].[{table_name}] ({len(headers)} 列)")
# 3. 插入数据
total = batch_insert(cursor, table_name, headers, merged, cfg, actual_pk)
@@ -414,6 +471,8 @@ def main():
parser = argparse.ArgumentParser(description='Excel to SQL Server Migration')
parser.add_argument('--config', default=None, help='配置文件路径 (默认: config.yaml)')
parser.add_argument('--dry-run', action='store_true', help='仅预览,不写入数据库')
parser.add_argument('--truncate', action='store_true',
help='使用 TRUNCATE 模式(表存在时截断而非重建)')
parser.add_argument('--table', nargs='+', default=None,
help='只迁移指定的表,例如 --table PG_2022 TM_2025')
args = parser.parse_args()
@@ -455,7 +514,8 @@ def main():
total_rows = 0
for i, table_cfg in enumerate(tables, 1):
print(f"\n[{i}/{total_tables}]", end='')
rows = migrate_table(cursor, table_cfg, cfg, dry_run=args.dry_run)
rows = migrate_table(cursor, table_cfg, cfg, dry_run=args.dry_run,
use_truncate=args.truncate)
total_rows += rows
if not args.dry_run: