AllForCompany/BLD_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove Excel migration pipeline; keep Access-only sync

Browse Source 
Drop the Excel (.xlsm) production-execution-card pipeline now that the
project only synchronizes Access databases to SQL Server.

- Delete excel_sync_to_sql.py, migration.py, config/field_mappings.py
- Remove Excel-only config symbols (EXCEL_CONFIGS, MIGRATION_TASKS,
  EXECUTION_CARD_FIELDS, CONTRACT_DATA_*, CACHE_DIR, TEMP_DIR,
  EXCEL_SYNC_* settings) from the config package
- Drop now-unused deps from requirements.txt: pandas, sqlalchemy, openpyxl
- Update .env.example, CLAUDE.md, and the uptime_kuma_utils docstring
- Fix the tube-bending workshop Access table mapping in SYNC_MAPPING
  (source table renamed; old name no longer exists)

The three Excel-sourced tables in warehouseOutbound (executionCardData,
contractData, customerProductType) and their data are left untouched.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-06-17 13:09:36 +08:00
parent f9b2d3b4da
commit f6580b4994
11 changed files with 231 additions and 944 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
config/__init__.py
View File

@@ -7,17 +7,13 @@ load_dotenv()
# 数据库配置
from .database import SQL_SERVER_CONFIG, SQL_SERVER_CONN, DB_CONFIG, ACCESS_DRIVER
# 文件路径配置
from .file_sources import SYNC_MAPPING, EXCEL_CONFIGS, MIGRATION_TASKS
# 字段映射配置
from .field_mappings import TABLE_SCHEMA, CONTRACT_MAPPING
# 文件路径配置（Access 同步映射）
from .file_sources import SYNC_MAPPING
# 应用设置
from .app_settings import (
LOG_TABLE_CONFIG, NTFY_CONFIG, UPTIME_KUMA_CONFIG, EXCEL_SYNC_UPTIME_KUMA_CONFIG,
POLL_INTERVAL, EXCEL_SYNC_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR,
EXECUTION_CARD_FIELDS, CONTRACT_DATA_FIELDS, CONTRACT_DATA_MAPPING
LOG_TABLE_CONFIG, NTFY_CONFIG, UPTIME_KUMA_CONFIG,
POLL_INTERVAL, BATCH_SIZE,
)
# 统一导出列表
@@ -25,11 +21,8 @@ __all__ = [
# Database
'SQL_SERVER_CONFIG', 'SQL_SERVER_CONN', 'DB_CONFIG', 'ACCESS_DRIVER',
# File Sources
'SYNC_MAPPING', 'EXCEL_CONFIGS', 'MIGRATION_TASKS',
# Field Mappings
'TABLE_SCHEMA', 'CONTRACT_MAPPING',
'SYNC_MAPPING',
# App Settings
'LOG_TABLE_CONFIG', 'NTFY_CONFIG', 'UPTIME_KUMA_CONFIG', 'EXCEL_SYNC_UPTIME_KUMA_CONFIG',
'POLL_INTERVAL', 'EXCEL_SYNC_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR',
'EXECUTION_CARD_FIELDS', 'CONTRACT_DATA_FIELDS', 'CONTRACT_DATA_MAPPING'
'LOG_TABLE_CONFIG', 'NTFY_CONFIG', 'UPTIME_KUMA_CONFIG',
'POLL_INTERVAL', 'BATCH_SIZE',
]

90
config/app_settings.py
View File

@@ -36,96 +36,6 @@ UPTIME_KUMA_CONFIG = {
'heartbeat_interval': 59 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致
}
# Excel 同步服务心跳
EXCEL_SYNC_UPTIME_KUMA_CONFIG = {
'enabled': True,
'push_url': os.environ.get('EXCEL_SYNC_UPTIME_KUMA_PUSH_URL', ''),
'heartbeat_interval': 59 # 心跳间隔(秒)，需要与 Uptime Kuma 设置一致
}
# ================= 运行参数 =================
# 合并原 config.py 和 update_config.py 的配置
POLL_INTERVAL = 30 # 轮询间隔(秒)
EXCEL_SYNC_INTERVAL = 600 # Excel 同步周期(秒)，默认 10 分钟
BATCH_SIZE = 10000 # 批量处理大小
CACHE_DIR = os.path.join(os.getcwd(), "temp") # Excel 缓存目录
TEMP_DIR = os.path.join(os.getcwd(), "temp") # 临时目录（兼容 migration.py）
# ================= 字段配置 =================
# 从原 update_config.py (sync_excel_to_sql.py) 抽取
EXECUTION_CARD_FIELDS = {
"合同年份": ("str", 10),
"总排号": ("str", 50),
"序号": ("int", None),
"订单号": ("str", 150),
"车间号": ("str", 20),
"销售内部号": ("str", 50),
"经办人": ("str", 20),
"签订日期": ("date", None),
"交货日期": ("date", None),
"客户名称": ("str", 200),
"产品名称": ("str", 200),
"客户型号": ("str", 200),
"选型型号": ("str", 200),
"量程": ("str", 150),
"数量": ("int", None),
"备注2": ("str", 500),
"备注1": ("str", 500),
"位号": ("str", 500),
"技术参数": ("str", 1000),
"车间": ("str", 200),
"工令号": ("str", 200),
"接单日期": ("date", None),
"新参数": ("str", 1000),
"基本型号": ("str", 200),
"公称外径": ("str", 200),
"安装代码": ("str", 200),
"设计形式": ("str", 200),
"技术安装代码": ("str", 200),
"隔膜类型": ("str", 200),
"标准": ("str", 100),
"隔膜大小": ("str", 200),
"隔膜材质": ("str", 200),
"膜片": ("str", 200),
"膜片材质": ("str", 200),
"CRM明细ID号": ("str", 200),
"成品物料编码": ("str", 200),
"工单号": ("str", 200),
"盘号": ("str", 200),
"编码": ("str", 200),
"型批名称": ("str", 200),
"型批型号": ("str", 200),
"开票名称": ("str", 200),
"开票型号": ("str", 200),
"上数据时间": ("date", None),
"生产代码1": ("str", 50),
"生产代码2": ("str", 50)
}
CONTRACT_DATA_FIELDS = {
"合同年份": ("str", 10),
"车间号": ("str", 20),
"工令号": ("str", 200),
"订单号": ("str", 150),
"客户名称": ("str", 200),
"产品型号": ("str", 200),
"量程": ("str", 150),
"数量": ("int", None),
"单价": ("int", None),
"ID": ("int", None),
"位号": ("str", 500)
}
CONTRACT_DATA_MAPPING = {
"合同年份": "合同年份",
"车间号": "车间号",
"工令号": "工令号",
"订单号": "订单号",
"客户名称": "客户名称",
"产品型号": "选型型号",
"量程": "量程",
"数量": "数量",
"单价": None,
"ID": None,
"位号": "位号"
}

73
config/field_mappings.py
View File

@@ -1,73 +0,0 @@
# config/field_mappings.py
# 字段映射和清洗规则配置
import os
from config.app_settings import EXECUTION_CARD_FIELDS
# ================= 字段清洗规则 =================
# 从原 update_config.py 抽取
# 基于 NVARCHAR (按字符数计算长度)
TABLE_SCHEMA = {
"合同年份": {"type": "str", "max_len": 10},
"总排号": {"type": "str", "max_len": 50},
"序号": {"type": "int"},
"订单号": {"type": "str", "max_len": 150},
"车间号": {"type": "str", "max_len": 20},
"销售内部号": {"type": "str", "max_len": 50},
"经办人": {"type": "str", "max_len": 20},
"签订日期": {"type": "date"},
"交货日期": {"type": "date"},
"客户名称": {"type": "str", "max_len": 200},
"产品名称": {"type": "str", "max_len": 200},
"客户型号": {"type": "str", "max_len": 200},
"选型型号": {"type": "str", "max_len": 200},
"量程": {"type": "str", "max_len": 150},
"数量": {"type": "int"},
"备注2": {"type": "str", "max_len": 500},
"备注1": {"type": "str", "max_len": 500},
"位号": {"type": "str", "max_len": 500},
"技术参数": {"type": "str", "max_len": 1000},
"车间": {"type": "str", "max_len": 200},
"工令号": {"type": "str", "max_len": 200},
"接单日期": {"type": "date"},
"新参数": {"type": "str", "max_len": 1000},
"基本型号": {"type": "str", "max_len": 200},
"公称外径": {"type": "str", "max_len": 200},
"安装代码": {"type": "str", "max_len": 200},
"设计形式": {"type": "str", "max_len": 200},
"技术安装代码": {"type": "str", "max_len": 200},
"隔膜类型": {"type": "str", "max_len": 200},
"标准": {"type": "str", "max_len": 100},
"隔膜大小": {"type": "str", "max_len": 200},
"隔膜材质": {"type": "str", "max_len": 200},
"膜片": {"type": "str", "max_len": 200},
"膜片材质": {"type": "str", "max_len": 200},
"CRM明细ID号": {"type": "str", "max_len": 200},
"成品物料编码": {"type": "str", "max_len": 200},
"工单号": {"type": "str", "max_len": 200},
"盘号": {"type": "str", "max_len": 200},
"编码": {"type": "str", "max_len": 200},
"型批名称": {"type": "str", "max_len": 200},
"型批型号": {"type": "str", "max_len": 200},
"开票名称": {"type": "str", "max_len": 200},
"开票型号": {"type": "str", "max_len": 200},
"上数据时间": {"type": "date"},
"生产代码1": {"type": "str", "max_len": 50},
"生产代码2": {"type": "str", "max_len": 50}
}
# ================= 合同数据映射 =================
# 从原 update_config.py 抽取
CONTRACT_MAPPING = {
"合同年份": "合同年份",
"车间号": "车间号",
"工令号": "工令号",
"订单号": "订单号",
"客户名称": "客户名称",
"产品型号": "选型型号",
"量程": "量程",
"数量": "数量",
"单价": None,
"ID": None,
"位号": "位号"
}

152
config/file_sources.py
View File

@@ -313,160 +313,10 @@ SYNC_MAPPING = {
}
},
r"\\192.168.110.114\生产进度表\2026年数据\弯管车间.accdb": {
"执行卡下发记录": {
"烘洗": {
"target_schema": "tubeBending",
"target_table": "烘洗_YEAR2026",
"pk_col": "ID"
}
}
}
# ================= Excel 文件配置 =================
# 从原 update_config.py 抽取
EXCEL_CONFIGS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2022",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2025",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2026",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
}
]
# ================= 迁移任务配置 =================
# 从原 migration.py 抽取
MIGRATION_TASKS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"year": 2022,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"year": 2025,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"year": 2026,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
}
]