AllForCompany/BLD_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance Excel synchronization by adding Uptime Kuma heartbeat functionality, new Excel sync configuration, and additional file mappings

Browse Source
This commit is contained in:
Misaka_Company
2026-01-14 09:13:45 +08:00
parent 3bcc5aafbe
commit 2a2ec22d1c
4 changed files with 108 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
config/__init__.py
View File

@@ -12,8 +12,8 @@ from .field_mappings import TABLE_SCHEMA, CONTRACT_MAPPING
# 应用设置
from .app_settings import (
LOG_TABLE_CONFIG, NTFY_CONFIG, UPTIME_KUMA_CONFIG,
POLL_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR,
LOG_TABLE_CONFIG, NTFY_CONFIG, UPTIME_KUMA_CONFIG, EXCEL_SYNC_UPTIME_KUMA_CONFIG,
POLL_INTERVAL, EXCEL_SYNC_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR,
EXECUTION_CARD_FIELDS, CONTRACT_DATA_FIELDS, CONTRACT_DATA_MAPPING
)
@@ -26,7 +26,7 @@ __all__ = [
# Field Mappings
'TABLE_SCHEMA', 'CONTRACT_MAPPING',
# App Settings
'LOG_TABLE_CONFIG', 'NTFY_CONFIG', 'UPTIME_KUMA_CONFIG',
'POLL_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR',
'LOG_TABLE_CONFIG', 'NTFY_CONFIG', 'UPTIME_KUMA_CONFIG', 'EXCEL_SYNC_UPTIME_KUMA_CONFIG',
'POLL_INTERVAL', 'EXCEL_SYNC_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR',
'EXECUTION_CARD_FIELDS', 'CONTRACT_DATA_FIELDS', 'CONTRACT_DATA_MAPPING'
]

11
config/app_settings.py
View File

@@ -29,15 +29,24 @@ NTFY_CONFIG = {
}
# ================= Uptime Kuma 心跳配置 =================
# 增量同步服务心跳
UPTIME_KUMA_CONFIG = {
'enabled': True,
'push_url': 'https://uptimekuma.server10086.icu/api/push/clMSgOQs4CJeF7DJAOiHaFaaZzoL8eB9',
'heartbeat_interval': 60 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致
'heartbeat_interval': 59 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致
}
# Excel 同步服务心跳
EXCEL_SYNC_UPTIME_KUMA_CONFIG = {
'enabled': True,
'push_url': 'https://uptimekuma.server10086.icu/api/push/MEXi5DYnC3nTMs2OfMqdelK3FawymtVY',
'heartbeat_interval': 59 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致
}
# ================= 运行参数 =================
# 合并原 config.py 和 update_config.py 的配置
POLL_INTERVAL = 5 # 轮询间隔(秒)
EXCEL_SYNC_INTERVAL = 600 # Excel 同步周期(秒)，默认 10 分钟
BATCH_SIZE = 10000 # 批量处理大小
CACHE_DIR = os.path.join(os.getcwd(), "temp") # Excel 缓存目录
TEMP_DIR = os.path.join(os.getcwd(), "temp") # 临时目录（兼容 migration.py）

5
config/file_sources.py
View File

@@ -86,6 +86,11 @@ SYNC_MAPPING = {
"target_schema": "machining",
"target_table": "喷涂寄出_YEAR2026",
"pk_col": "ID"
},
"车波纹": {
"target_schema": "machining",
"target_table": "车波纹_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\计划.accdb": {

99
etl_manager.py → excel_sync_to_sql.py
View File

@@ -6,6 +6,8 @@ import argparse
import datetime
import urllib.parse
import warnings
import time
import requests
import pandas as pd
import numpy as np
from sqlalchemy import create_engine, text
@@ -14,8 +16,9 @@ from sqlalchemy.types import NVARCHAR, Integer, Date
# 导入配置
from log_utils import (log_error, log_warning, log_info, log_processing, log_file, log_sync,
log_start, log_complete, LoggerManager)
from config import DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA
log_start, log_complete, log_stop, LoggerManager)
from config import (DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA,
EXCEL_SYNC_INTERVAL, EXCEL_SYNC_UPTIME_KUMA_CONFIG)
# ================= 抑制 openpyxl 的数据验证警告 =================
warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl')
@@ -291,25 +294,101 @@ class DataSynchronizer:
except Exception as e:
log_error(f"生成 ContractData 失败: {e}", exc_info=True)
# ================= Uptime Kuma 心跳 =================
_last_heartbeat_time = 0
def send_heartbeat():
"""发送心跳信号到 Uptime Kuma"""
global _last_heartbeat_time
if not EXCEL_SYNC_UPTIME_KUMA_CONFIG.get('enabled', False):
return
try:
url = EXCEL_SYNC_UPTIME_KUMA_CONFIG['push_url']
params = {
'status': 'up',
'msg': 'OK',
'ping': ''
}
response = requests.get(url, params=params, timeout=5)
response.raise_for_status()
_last_heartbeat_time = time.time()
except Exception as e:
log_warning(f"心跳发送失败: {e}")
def main():
# 初始化日志管理器
LoggerManager("etl_manager", log_prefix="sync")
LoggerManager("excel_sync", log_prefix="excel_sync")
# 解析参数
parser = argparse.ArgumentParser(description="Excel数据同步至SQL Server")
parser.add_argument('--force', action='store_true', help='强制同步所有文件')
parser.add_argument('--once', action='store_true', help='只运行一次后退出')
args = parser.parse_args()
syncer = DataSynchronizer(force_sync=args.force)
if args.force:
log_start("Excel 同步任务 (强制模式)")
else:
log_start("Excel 同步任务 (增量模式)")
# 启动信息
mode = "强制模式" if args.force else "增量模式"
if args.once:
log_start(f"Excel 同步任务 ({mode}, 单次运行)")
syncer.process_excel_files()
syncer.generate_contract_data()
log_complete("Excel 同步任务已完成")
return
syncer.process_excel_files()
syncer.generate_contract_data()
# 周期性运行模式
log_start(f"Excel 同步服务已启动 ({mode})")
log_info(f"同步周期: {EXCEL_SYNC_INTERVAL} 秒 ({EXCEL_SYNC_INTERVAL//60} 分钟)")
if EXCEL_SYNC_UPTIME_KUMA_CONFIG.get('enabled', False):
log_info(f"心跳间隔: {EXCEL_SYNC_UPTIME_KUMA_CONFIG['heartbeat_interval']} 秒")
log_info("=" * 70)
log_complete("Excel 同步任务已完成")
# 启动时发送第一次心跳
send_heartbeat()
try:
while True:
try:
# 执行同步任务
log_info(f"开始执行周期性同步检查...")
syncer.process_excel_files()
syncer.generate_contract_data()
log_info(f"周期性同步检查完成")
# 下次同步时间
next_sync_time = time.time() + EXCEL_SYNC_INTERVAL
log_info(f"下次同步将在 {EXCEL_SYNC_INTERVAL//60} 分钟后进行")
# 等待下次同步，期间持续发送心跳
while time.time() < next_sync_time:
# 检查是否需要发送心跳
time_since_last_heartbeat = time.time() - _last_heartbeat_time
if time_since_last_heartbeat >= EXCEL_SYNC_UPTIME_KUMA_CONFIG['heartbeat_interval']:
send_heartbeat()
# 短暂休眠
time.sleep(1)
except KeyboardInterrupt:
log_info("=" * 70)
log_stop("收到停止信号，服务正在关闭...")
break
except Exception as e:
log_error(f"同步任务异常: {e}", exc_info=True)
log_info(f"将在 {EXCEL_SYNC_INTERVAL//60} 分钟后重试...")
time.sleep(EXCEL_SYNC_INTERVAL)
finally:
# 停止时发送心跳停止信号
if EXCEL_SYNC_UPTIME_KUMA_CONFIG.get('enabled', False):
try:
url = EXCEL_SYNC_UPTIME_KUMA_CONFIG['push_url']
params = {'status': 'down', 'msg': 'Service stopped'}
requests.get(url, params=params, timeout=5)
except:
pass
if __name__ == "__main__":
main()