diff --git a/config/__init__.py b/config/__init__.py index ed9449a..ff8e21b 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -12,8 +12,8 @@ from .field_mappings import TABLE_SCHEMA, CONTRACT_MAPPING # 应用设置 from .app_settings import ( - LOG_TABLE_CONFIG, NTFY_CONFIG, UPTIME_KUMA_CONFIG, - POLL_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR, + LOG_TABLE_CONFIG, NTFY_CONFIG, UPTIME_KUMA_CONFIG, EXCEL_SYNC_UPTIME_KUMA_CONFIG, + POLL_INTERVAL, EXCEL_SYNC_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR, EXECUTION_CARD_FIELDS, CONTRACT_DATA_FIELDS, CONTRACT_DATA_MAPPING ) @@ -26,7 +26,7 @@ __all__ = [ # Field Mappings 'TABLE_SCHEMA', 'CONTRACT_MAPPING', # App Settings - 'LOG_TABLE_CONFIG', 'NTFY_CONFIG', 'UPTIME_KUMA_CONFIG', - 'POLL_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR', + 'LOG_TABLE_CONFIG', 'NTFY_CONFIG', 'UPTIME_KUMA_CONFIG', 'EXCEL_SYNC_UPTIME_KUMA_CONFIG', + 'POLL_INTERVAL', 'EXCEL_SYNC_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR', 'EXECUTION_CARD_FIELDS', 'CONTRACT_DATA_FIELDS', 'CONTRACT_DATA_MAPPING' ] diff --git a/config/app_settings.py b/config/app_settings.py index 744e7f3..7469f5c 100644 --- a/config/app_settings.py +++ b/config/app_settings.py @@ -29,15 +29,24 @@ NTFY_CONFIG = { } # ================= Uptime Kuma 心跳配置 ================= +# 增量同步服务心跳 UPTIME_KUMA_CONFIG = { 'enabled': True, 'push_url': 'https://uptimekuma.server10086.icu/api/push/clMSgOQs4CJeF7DJAOiHaFaaZzoL8eB9', - 'heartbeat_interval': 60 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致 + 'heartbeat_interval': 59 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致 +} + +# Excel 同步服务心跳 +EXCEL_SYNC_UPTIME_KUMA_CONFIG = { + 'enabled': True, + 'push_url': 'https://uptimekuma.server10086.icu/api/push/MEXi5DYnC3nTMs2OfMqdelK3FawymtVY', + 'heartbeat_interval': 59 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致 } # ================= 运行参数 ================= # 合并原 config.py 和 update_config.py 的配置 POLL_INTERVAL = 5 # 轮询间隔(秒) +EXCEL_SYNC_INTERVAL = 600 # Excel 同步周期(秒)，默认 10 分钟 BATCH_SIZE = 10000 # 批量处理大小 CACHE_DIR = os.path.join(os.getcwd(), "temp") # Excel 缓存目录 TEMP_DIR = os.path.join(os.getcwd(), "temp") # 临时目录（兼容 migration.py） diff --git a/config/file_sources.py b/config/file_sources.py index 73a417d..3ddfb6a 100644 --- a/config/file_sources.py +++ b/config/file_sources.py @@ -86,6 +86,11 @@ SYNC_MAPPING = { "target_schema": "machining", "target_table": "喷涂寄出_YEAR2026", "pk_col": "ID" + }, + "车波纹": { + "target_schema": "machining", + "target_table": "车波纹_YEAR2026", + "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\计划.accdb": { diff --git a/etl_manager.py b/excel_sync_to_sql.py similarity index 78% rename from etl_manager.py rename to excel_sync_to_sql.py index 0932296..aeb7ea1 100644 --- a/etl_manager.py +++ b/excel_sync_to_sql.py @@ -6,6 +6,8 @@ import argparse import datetime import urllib.parse import warnings +import time +import requests import pandas as pd import numpy as np from sqlalchemy import create_engine, text @@ -14,8 +16,9 @@ from sqlalchemy.types import NVARCHAR, Integer, Date # 导入配置 from log_utils import (log_error, log_warning, log_info, log_processing, log_file, log_sync, - log_start, log_complete, LoggerManager) -from config import DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA + log_start, log_complete, log_stop, LoggerManager) +from config import (DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA, + EXCEL_SYNC_INTERVAL, EXCEL_SYNC_UPTIME_KUMA_CONFIG) # ================= 抑制 openpyxl 的数据验证警告 ================= warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl') @@ -291,25 +294,101 @@ class DataSynchronizer: except Exception as e: log_error(f"生成 ContractData 失败: {e}", exc_info=True) +# ================= Uptime Kuma 心跳 ================= + +_last_heartbeat_time = 0 + +def send_heartbeat(): + """发送心跳信号到 Uptime Kuma""" + global _last_heartbeat_time + + if not EXCEL_SYNC_UPTIME_KUMA_CONFIG.get('enabled', False): + return + + try: + url = EXCEL_SYNC_UPTIME_KUMA_CONFIG['push_url'] + params = { + 'status': 'up', + 'msg': 'OK', + 'ping': '' + } + response = requests.get(url, params=params, timeout=5) + response.raise_for_status() + _last_heartbeat_time = time.time() + except Exception as e: + log_warning(f"心跳发送失败: {e}") + def main(): # 初始化日志管理器 - LoggerManager("etl_manager", log_prefix="sync") + LoggerManager("excel_sync", log_prefix="excel_sync") + # 解析参数 parser = argparse.ArgumentParser(description="Excel数据同步至SQL Server") parser.add_argument('--force', action='store_true', help='强制同步所有文件') + parser.add_argument('--once', action='store_true', help='只运行一次后退出') args = parser.parse_args() syncer = DataSynchronizer(force_sync=args.force) - if args.force: - log_start("Excel 同步任务 (强制模式)") - else: - log_start("Excel 同步任务 (增量模式)") + # 启动信息 + mode = "强制模式" if args.force else "增量模式" + if args.once: + log_start(f"Excel 同步任务 ({mode}, 单次运行)") + syncer.process_excel_files() + syncer.generate_contract_data() + log_complete("Excel 同步任务已完成") + return - syncer.process_excel_files() - syncer.generate_contract_data() + # 周期性运行模式 + log_start(f"Excel 同步服务已启动 ({mode})") + log_info(f"同步周期: {EXCEL_SYNC_INTERVAL} 秒 ({EXCEL_SYNC_INTERVAL//60} 分钟)") + if EXCEL_SYNC_UPTIME_KUMA_CONFIG.get('enabled', False): + log_info(f"心跳间隔: {EXCEL_SYNC_UPTIME_KUMA_CONFIG['heartbeat_interval']} 秒") + log_info("=" * 70) - log_complete("Excel 同步任务已完成") + # 启动时发送第一次心跳 + send_heartbeat() + + try: + while True: + try: + # 执行同步任务 + log_info(f"开始执行周期性同步检查...") + syncer.process_excel_files() + syncer.generate_contract_data() + log_info(f"周期性同步检查完成") + + # 下次同步时间 + next_sync_time = time.time() + EXCEL_SYNC_INTERVAL + log_info(f"下次同步将在 {EXCEL_SYNC_INTERVAL//60} 分钟后进行") + + # 等待下次同步，期间持续发送心跳 + while time.time() < next_sync_time: + # 检查是否需要发送心跳 + time_since_last_heartbeat = time.time() - _last_heartbeat_time + if time_since_last_heartbeat >= EXCEL_SYNC_UPTIME_KUMA_CONFIG['heartbeat_interval']: + send_heartbeat() + + # 短暂休眠 + time.sleep(1) + + except KeyboardInterrupt: + log_info("=" * 70) + log_stop("收到停止信号，服务正在关闭...") + break + except Exception as e: + log_error(f"同步任务异常: {e}", exc_info=True) + log_info(f"将在 {EXCEL_SYNC_INTERVAL//60} 分钟后重试...") + time.sleep(EXCEL_SYNC_INTERVAL) + finally: + # 停止时发送心跳停止信号 + if EXCEL_SYNC_UPTIME_KUMA_CONFIG.get('enabled', False): + try: + url = EXCEL_SYNC_UPTIME_KUMA_CONFIG['push_url'] + params = {'status': 'down', 'msg': 'Service stopped'} + requests.get(url, params=params, timeout=5) + except: + pass if __name__ == "__main__": main() \ No newline at end of file