AllForCompany/BLD_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor configuration management and remove deprecated files

Browse Source 
- Consolidated database, file source, and field mapping configurations into dedicated modules under the `config` directory.
- Removed hardcoded database connection details from `migration.py` and replaced them with imports from the new configuration structure.
- Updated `ntfy_utils.py` and `run_incremental_sync.py` to utilize the new configuration imports for cleaner code and better maintainability.
- Deleted `update_config.py` as its contents have been integrated into the new configuration files.
- Added a new `settings.local.json` for managing permissions related to script execution.
- Enhanced the structure of the migration tasks and Excel configurations for better organization and clarity.
This commit is contained in:
Misaka_Company
2026-01-12 12:49:51 +08:00
parent ee14de0435
commit 83c90f0161
13 changed files with 501 additions and 365 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
config/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
# config/__init__.py
# 统一配置导出接口
# 数据库配置
from .database import SQL_SERVER_CONFIG, SQL_SERVER_CONN, DB_CONFIG, ACCESS_DRIVER
# 文件路径配置
from .file_sources import SYNC_MAPPING, EXCEL_CONFIGS, MIGRATION_TASKS
# 字段映射配置
from .field_mappings import TABLE_SCHEMA, CONTRACT_MAPPING
# 应用设置
from .app_settings import (
LOG_TABLE_CONFIG, NTFY_CONFIG,
POLL_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR,
EXECUTION_CARD_FIELDS, CONTRACT_DATA_FIELDS, CONTRACT_DATA_MAPPING
)
# 统一导出列表
__all__ = [
# Database
'SQL_SERVER_CONFIG', 'SQL_SERVER_CONN', 'DB_CONFIG', 'ACCESS_DRIVER',
# File Sources
'SYNC_MAPPING', 'EXCEL_CONFIGS', 'MIGRATION_TASKS',
# Field Mappings
'TABLE_SCHEMA', 'CONTRACT_MAPPING',
# App Settings
'LOG_TABLE_CONFIG', 'NTFY_CONFIG',
'POLL_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR',
'EXECUTION_CARD_FIELDS', 'CONTRACT_DATA_FIELDS', 'CONTRACT_DATA_MAPPING'
]

115
config/app_settings.py Normal file
View File

@@ -0,0 +1,115 @@
# config/app_settings.py
# 应用设置配置
import os
# ================= 日志表配置 =================
# 从原 config.py 抽取
LOG_TABLE_CONFIG = {
'schema': 'dbo',
'table_name': 'TableChangeLog',
'col_log_id': 'LogID',
'col_table_name': 'TableName',
'col_record_id': 'RecordID',
'col_address': 'TableAddress',
'col_synced': 'Synced'
}
# ================= ntfy 配置 =================
# 从原 config.py 抽取
NTFY_CONFIG = {
'enabled': True,
'server_url': 'https://ntfy.server10086.icu',
'topic': 'bld',
'token': 'tk_eop5fs66acxtwxf6vlkiojhdvkgb0',
'priority': {
'error': 'high',
'critical': 'urgent'
}
}
# ================= 运行参数 =================
# 合并原 config.py 和 update_config.py 的配置
POLL_INTERVAL = 5 # 轮询间隔(秒)
BATCH_SIZE = 10000 # 批量处理大小
CACHE_DIR = os.path.join(os.getcwd(), "temp") # Excel 缓存目录
TEMP_DIR = os.path.join(os.getcwd(), "temp") # 临时目录（兼容 migration.py）
# ================= 字段配置 =================
# 从原 update_config.py (sync_excel_to_sql.py) 抽取
EXECUTION_CARD_FIELDS = {
"合同年份": ("str", 10),
"总排号": ("str", 50),
"序号": ("int", None),
"订单号": ("str", 150),
"车间号": ("str", 20),
"销售内部号": ("str", 50),
"经办人": ("str", 20),
"签订日期": ("date", None),
"交货日期": ("date", None),
"客户名称": ("str", 200),
"产品名称": ("str", 200),
"客户型号": ("str", 200),
"选型型号": ("str", 200),
"量程": ("str", 150),
"数量": ("int", None),
"备注2": ("str", 500),
"备注1": ("str", 500),
"位号": ("str", 500),
"技术参数": ("str", 1000),
"车间": ("str", 200),
"工令号": ("str", 200),
"接单日期": ("date", None),
"新参数": ("str", 1000),
"基本型号": ("str", 200),
"公称外径": ("str", 200),
"安装代码": ("str", 200),
"设计形式": ("str", 200),
"技术安装代码": ("str", 200),
"隔膜类型": ("str", 200),
"标准": ("str", 100),
"隔膜大小": ("str", 200),
"隔膜材质": ("str", 200),
"膜片": ("str", 200),
"膜片材质": ("str", 200),
"CRM明细ID号": ("str", 200),
"成品物料编码": ("str", 200),
"工单号": ("str", 200),
"盘号": ("str", 200),
"编码": ("str", 200),
"型批名称": ("str", 200),
"型批型号": ("str", 200),
"开票名称": ("str", 200),
"开票型号": ("str", 200),
"上数据时间": ("date", None),
"生产代码1": ("str", 50),
"生产代码2": ("str", 50)
}
CONTRACT_DATA_FIELDS = {
"合同年份": ("str", 10),
"车间号": ("str", 20),
"工令号": ("str", 200),
"订单号": ("str", 150),
"客户名称": ("str", 200),
"产品型号": ("str", 200),
"量程": ("str", 150),
"数量": ("int", None),
"单价": ("int", None),
"ID": ("int", None),
"位号": ("str", 500)
}
CONTRACT_DATA_MAPPING = {
"合同年份": "合同年份",
"车间号": "车间号",
"工令号": "工令号",
"订单号": "订单号",
"客户名称": "客户名称",
"产品型号": "选型型号",
"量程": "量程",
"数量": "数量",
"单价": None,
"ID": None,
"位号": "位号"
}

30
config/database.py Normal file
View File

@@ -0,0 +1,30 @@
# config/database.py
# 统一的数据库配置
# ================= SQL Server 配置 =================
SQL_SERVER_CONFIG = {
'driver': 'ODBC Driver 18 for SQL Server',
'server': '192.168.110.114',
'database': 'CompanyDB',
'username': 'peng',
'password': 'Cqbld123456.',
'trust_server_certificate': 'yes'
}
# ================= Access 驱动配置 =================
ACCESS_DRIVER = "{Microsoft Access Driver (*.mdb, *.accdb)}"
# ================= 兼容性别名 =================
# 为了兼容旧代码，提供不同命名风格的别名
# SQL_SERVER_CONN - 兼容 db_utils.py (使用 uid/pwd)
SQL_SERVER_CONN = {
'driver': f"{{{SQL_SERVER_CONFIG['driver']}}}",
'server': SQL_SERVER_CONFIG['server'],
'database': SQL_SERVER_CONFIG['database'],
'uid': SQL_SERVER_CONFIG['username'],
'pwd': SQL_SERVER_CONFIG['password']
}
# DB_CONFIG - 兼容 etl_manager.py, migration.py (使用 username/password)
DB_CONFIG = SQL_SERVER_CONFIG.copy()

73
config/field_mappings.py Normal file
View File

@@ -0,0 +1,73 @@
# config/field_mappings.py
# 字段映射和清洗规则配置
import os
from config.app_settings import EXECUTION_CARD_FIELDS
# ================= 字段清洗规则 =================
# 从原 update_config.py 抽取
# 基于 NVARCHAR (按字符数计算长度)
TABLE_SCHEMA = {
"合同年份": {"type": "str", "max_len": 10},
"总排号": {"type": "str", "max_len": 50},
"序号": {"type": "int"},
"订单号": {"type": "str", "max_len": 150},
"车间号": {"type": "str", "max_len": 20},
"销售内部号": {"type": "str", "max_len": 50},
"经办人": {"type": "str", "max_len": 20},
"签订日期": {"type": "date"},
"交货日期": {"type": "date"},
"客户名称": {"type": "str", "max_len": 200},
"产品名称": {"type": "str", "max_len": 200},
"客户型号": {"type": "str", "max_len": 200},
"选型型号": {"type": "str", "max_len": 200},
"量程": {"type": "str", "max_len": 150},
"数量": {"type": "int"},
"备注2": {"type": "str", "max_len": 500},
"备注1": {"type": "str", "max_len": 500},
"位号": {"type": "str", "max_len": 500},
"技术参数": {"type": "str", "max_len": 1000},
"车间": {"type": "str", "max_len": 200},
"工令号": {"type": "str", "max_len": 200},
"接单日期": {"type": "date"},
"新参数": {"type": "str", "max_len": 1000},
"基本型号": {"type": "str", "max_len": 200},
"公称外径": {"type": "str", "max_len": 200},
"安装代码": {"type": "str", "max_len": 200},
"设计形式": {"type": "str", "max_len": 200},
"技术安装代码": {"type": "str", "max_len": 200},
"隔膜类型": {"type": "str", "max_len": 200},
"标准": {"type": "str", "max_len": 100},
"隔膜大小": {"type": "str", "max_len": 200},
"隔膜材质": {"type": "str", "max_len": 200},
"膜片": {"type": "str", "max_len": 200},
"膜片材质": {"type": "str", "max_len": 200},
"CRM明细ID号": {"type": "str", "max_len": 200},
"成品物料编码": {"type": "str", "max_len": 200},
"工单号": {"type": "str", "max_len": 200},
"盘号": {"type": "str", "max_len": 200},
"编码": {"type": "str", "max_len": 200},
"型批名称": {"type": "str", "max_len": 200},
"型批型号": {"type": "str", "max_len": 200},
"开票名称": {"type": "str", "max_len": 200},
"开票型号": {"type": "str", "max_len": 200},
"上数据时间": {"type": "date"},
"生产代码1": {"type": "str", "max_len": 50},
"生产代码2": {"type": "str", "max_len": 50}
}
# ================= 合同数据映射 =================
# 从原 update_config.py 抽取
CONTRACT_MAPPING = {
"合同年份": "合同年份",
"车间号": "车间号",
"工令号": "工令号",
"订单号": "订单号",
"客户名称": "客户名称",
"产品型号": "选型型号",
"量程": "量程",
"数量": "数量",
"单价": None,
"ID": None,
"位号": "位号"
}

450
config/file_sources.py Normal file
View File

@@ -0,0 +1,450 @@
# config/file_sources.py
# 文件路径配置 - 从原 config.py 和 update_config.py 抽离
# ================= Access 同步映射配置 =================
# 从原 config.py 抽取
# 结构： { "Access文件绝对路径": { "Access表名": { SQL目标配置 } } }
SYNC_MAPPING = {
# 第一个 Access 文件
r"\\192.168.110.114\生产进度表\2025年数据\生产合同数据.accdb": {
"26年压力表合同数据": {
"target_schema": "productionContractData",
"target_table": "26年压力表合同数据",
"pk_col": "ID"
},
"26年温度计合同数据": {
"target_schema": "productionContractData",
"target_table": "26年温度计合同数据",
"pk_col": "ID"
},
"25年压力表合同数据": {
"target_schema": "productionContractData",
"target_table": "25年压力表合同数据",
"pk_col": "ID"
},
"25年温度计合同数据": {
"target_schema": "productionContractData",
"target_table": "25年温度计合同数据",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\成品入库.accdb": {
"成品交检记录": {
"target_schema": "productWarehousing",
"target_table": "成品交检记录_YEAR2026",
"pk_col": "ID"
},
"成品入库记录": {
"target_schema": "productWarehousing",
"target_table": "成品入库记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\一车间.accdb": {
"一车间记录": {
"target_schema": "workshopOne",
"target_table": "一车间记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\二车间.accdb": {
"二车间记录": {
"target_schema": "workshopTwo",
"target_table": "二车间记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\三车间.accdb": {
"三车间记录": {
"target_schema": "workshopThree",
"target_table": "三车间记录_YEAR2026",
"pk_col": "ID"
},
"温度计调校记录": {
"target_schema": "workshopThree",
"target_table": "温度计调校记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\机加工.accdb": {
"隔膜机加接收": {
"target_schema": "machining",
"target_table": "隔膜机加接收_YEAR2026",
"pk_col": "ID"
},
"膜片焊接": {
"target_schema": "machining",
"target_table": "膜片焊接_YEAR2026",
"pk_col": "ID"
},
"膜片接收": {
"target_schema": "machining",
"target_table": "膜片接收_YEAR2026",
"pk_col": "ID"
},
"喷涂寄出": {
"target_schema": "machining",
"target_table": "喷涂寄出_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\计划.accdb": {
"报警器接单记录": {
"target_schema": "contractPlanning",
"target_table": "报警器接单记录_YEAR2026",
"pk_col": "ID"
},
"二车间接单记录": {
"target_schema": "contractPlanning",
"target_table": "二车间接单记录_YEAR2026",
"pk_col": "ID"
},
"隔膜接单记录": {
"target_schema": "contractPlanning",
"target_table": "隔膜接单记录_YEAR2026",
"pk_col": "ID"
},
"过压保护器接单记录": {
"target_schema": "contractPlanning",
"target_table": "过压保护器接单记录_YEAR2026",
"pk_col": "ID"
},
"精密表部件接收记录": {
"target_schema": "contractPlanning",
"target_table": "精密表部件接收记录_YEAR2026",
"pk_col": "ID"
},
"精密表接单记录": {
"target_schema": "contractPlanning",
"target_table": "精密表接单记录_YEAR2026",
"pk_col": "ID"
},
"库房接单记录": {
"target_schema": "contractPlanning",
"target_table": "库房接单记录_YEAR2026",
"pk_col": "ID"
},
"软管接单记录": {
"target_schema": "contractPlanning",
"target_table": "软管接单记录_YEAR2026",
"pk_col": "ID"
},
"氩弧焊接单记录": {
"target_schema": "contractPlanning",
"target_table": "氩弧焊接单记录_YEAR2026",
"pk_col": "ID"
},
"一车间接单记录": {
"target_schema": "contractPlanning",
"target_table": "一车间接单记录_YEAR2026",
"pk_col": "ID"
},
"阻尼器接单记录": {
"target_schema": "contractPlanning",
"target_table": "阻尼器接单记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\检验记录数据库.accdb": {
"检验合格记录表": {
"target_schema": "inspectionRecords",
"target_table": "检验合格记录表_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\零件库.accdb": {
"表盘缺件记录": {
"target_schema": "partsWarehouse",
"target_table": "表盘缺件记录_YEAR2026",
"pk_col": "ID"
},
"表盘入库记录": {
"target_schema": "partsWarehouse",
"target_table": "表盘入库记录_YEAR2026",
"pk_col": "ID"
},
"部件发放记录": {
"target_schema": "partsWarehouse",
"target_table": "部件发放记录_YEAR2026",
"pk_col": "ID"
},
"出库单发出记录": {
"target_schema": "partsWarehouse",
"target_table": "出库单发出记录_YEAR2026",
"pk_col": "ID"
},
"出库单记录": {
"target_schema": "partsWarehouse",
"target_table": "出库单记录_YEAR2026",
"pk_col": "ID"
},
"法兰缺件记录": {
"target_schema": "partsWarehouse",
"target_table": "法兰缺件记录_YEAR2026",
"pk_col": "ID"
},
"法兰入库记录": {
"target_schema": "partsWarehouse",
"target_table": "法兰入库记录_YEAR2026",
"pk_col": "ID"
},
"卡到库房记录": {
"target_schema": "partsWarehouse",
"target_table": "卡到库房记录_YEAR2026",
"pk_col": "ID"
},
"喷涂到厂": {
"target_schema": "partsWarehouse",
"target_table": "喷涂到厂_YEAR2026",
"pk_col": "ID"
},
"缺件记录": {
"target_schema": "partsWarehouse",
"target_table": "缺件记录_YEAR2026",
"pk_col": "ID"
},
"温度计法兰缺件记录": {
"target_schema": "partsWarehouse",
"target_table": "温度计法兰缺件记录_YEAR2026",
"pk_col": "ID"
},
"温度计法兰入库记录": {
"target_schema": "partsWarehouse",
"target_table": "温度计法兰入库记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\温度计记录.accdb": {
"采 技 机记录": {
"target_schema": "thermometerRecord",
"target_table": "采 技 机记录_YEAR2026",
"pk_col": "ID"
},
"温度计调校记录": {
"target_schema": "thermometerRecord",
"target_table": "温度计调校记录_YEAR2026",
"pk_col": "ID"
},
"温度计检验记录": {
"target_schema": "thermometerRecord",
"target_table": "温度计检验记录_YEAR2026",
"pk_col": "ID"
},
"温度计组装记录": {
"target_schema": "thermometerRecord",
"target_table": "温度计组装记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\锡焊数据.accdb": {
"操作者1完成记录": {
"target_schema": "solderingData",
"target_table": "操作者1完成记录_YEAR2026",
"pk_col": "ID"
},
"操作者2完成记录": {
"target_schema": "solderingData",
"target_table": "操作者2完成记录_YEAR2026",
"pk_col": "ID"
},
"零件到车间记录": {
"target_schema": "solderingData",
"target_table": "零件到车间记录_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\氩弧焊.accdb": {
"表壳焊接记录": {
"target_schema": "TIGWelding",
"target_table": "表壳焊接记录_YEAR2026",
"pk_col": "ID"
},
"超压": {
"target_schema": "TIGWelding",
"target_table": "超压_YEAR2026",
"pk_col": "ID"
},
"氦测": {
"target_schema": "TIGWelding",
"target_table": "氦测_YEAR2026",
"pk_col": "ID"
},
"接收": {
"target_schema": "TIGWelding",
"target_table": "接收_YEAR2026",
"pk_col": "ID"
},
"退火": {
"target_schema": "TIGWelding",
"target_table": "退火_YEAR2026",
"pk_col": "ID"
},
"氩弧焊记录": {
"target_schema": "TIGWelding",
"target_table": "氩弧焊记录_YEAR2026",
"pk_col": "ID"
},
"氩弧焊领料": {
"target_schema": "TIGWelding",
"target_table": "氩弧焊领料_YEAR2026",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\执行卡下发记录.accdb": {
"执行卡下发记录": {
"target_schema": "executionCardIssuanceRecord",
"target_table": "执行卡下发记录_YEAR2026",
"pk_col": "ID"
}
}
}
# ================= Excel 文件配置 =================
# 从原 update_config.py 抽取
EXCEL_CONFIGS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2022",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2025",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2026",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
}
]
# ================= 迁移任务配置 =================
# 从原 migration.py 抽取
MIGRATION_TASKS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"year": 2022,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"year": 2025,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"year": 2026,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
}
]