diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..2a56823 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,13 @@ +{ + "permissions": { + "allow": [ + "Bash(python etl_manager.py --help)", + "Bash(python run_incremental_sync.py)", + "Bash(python -c \"import run_incremental_sync; print\\(''run_incremental_sync.py: import OK''\\)\")", + "Bash(python -c \"import etl_manager; print\\(''etl_manager.py: import OK''\\)\")", + "Bash(python -c \"import init_full_sync; print\\(''init_full_sync.py: import OK''\\)\")", + "Bash(python -c \"import migration; print\\(''migration.py: import OK''\\)\")", + "Bash(python migration.py)" + ] + } +} diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..ff1d8a2 --- /dev/null +++ b/config/__init__.py @@ -0,0 +1,32 @@ +# config/__init__.py +# 统一配置导出接口 + +# 数据库配置 +from .database import SQL_SERVER_CONFIG, SQL_SERVER_CONN, DB_CONFIG, ACCESS_DRIVER + +# 文件路径配置 +from .file_sources import SYNC_MAPPING, EXCEL_CONFIGS, MIGRATION_TASKS + +# 字段映射配置 +from .field_mappings import TABLE_SCHEMA, CONTRACT_MAPPING + +# 应用设置 +from .app_settings import ( + LOG_TABLE_CONFIG, NTFY_CONFIG, + POLL_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR, + EXECUTION_CARD_FIELDS, CONTRACT_DATA_FIELDS, CONTRACT_DATA_MAPPING +) + +# 统一导出列表 +__all__ = [ + # Database + 'SQL_SERVER_CONFIG', 'SQL_SERVER_CONN', 'DB_CONFIG', 'ACCESS_DRIVER', + # File Sources + 'SYNC_MAPPING', 'EXCEL_CONFIGS', 'MIGRATION_TASKS', + # Field Mappings + 'TABLE_SCHEMA', 'CONTRACT_MAPPING', + # App Settings + 'LOG_TABLE_CONFIG', 'NTFY_CONFIG', + 'POLL_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR', + 'EXECUTION_CARD_FIELDS', 'CONTRACT_DATA_FIELDS', 'CONTRACT_DATA_MAPPING' +] diff --git a/config/app_settings.py b/config/app_settings.py new file mode 100644 index 0000000..7f73d5c --- /dev/null +++ b/config/app_settings.py @@ -0,0 +1,115 @@ +# config/app_settings.py +# 应用设置配置 + +import os + +# ================= 日志表配置 ================= +# 从原 config.py 抽取 +LOG_TABLE_CONFIG = { + 'schema': 'dbo', + 'table_name': 'TableChangeLog', + 'col_log_id': 'LogID', + 'col_table_name': 'TableName', + 'col_record_id': 'RecordID', + 'col_address': 'TableAddress', + 'col_synced': 'Synced' +} + +# ================= ntfy 配置 ================= +# 从原 config.py 抽取 +NTFY_CONFIG = { + 'enabled': True, + 'server_url': 'https://ntfy.server10086.icu', + 'topic': 'bld', + 'token': 'tk_eop5fs66acxtwxf6vlkiojhdvkgb0', + 'priority': { + 'error': 'high', + 'critical': 'urgent' + } +} + +# ================= 运行参数 ================= +# 合并原 config.py 和 update_config.py 的配置 +POLL_INTERVAL = 5 # 轮询间隔(秒) +BATCH_SIZE = 10000 # 批量处理大小 +CACHE_DIR = os.path.join(os.getcwd(), "temp") # Excel 缓存目录 +TEMP_DIR = os.path.join(os.getcwd(), "temp") # 临时目录（兼容 migration.py） + +# ================= 字段配置 ================= +# 从原 update_config.py (sync_excel_to_sql.py) 抽取 +EXECUTION_CARD_FIELDS = { + "合同年份": ("str", 10), + "总排号": ("str", 50), + "序号": ("int", None), + "订单号": ("str", 150), + "车间号": ("str", 20), + "销售内部号": ("str", 50), + "经办人": ("str", 20), + "签订日期": ("date", None), + "交货日期": ("date", None), + "客户名称": ("str", 200), + "产品名称": ("str", 200), + "客户型号": ("str", 200), + "选型型号": ("str", 200), + "量程": ("str", 150), + "数量": ("int", None), + "备注2": ("str", 500), + "备注1": ("str", 500), + "位号": ("str", 500), + "技术参数": ("str", 1000), + "车间": ("str", 200), + "工令号": ("str", 200), + "接单日期": ("date", None), + "新参数": ("str", 1000), + "基本型号": ("str", 200), + "公称外径": ("str", 200), + "安装代码": ("str", 200), + "设计形式": ("str", 200), + "技术安装代码": ("str", 200), + "隔膜类型": ("str", 200), + "标准": ("str", 100), + "隔膜大小": ("str", 200), + "隔膜材质": ("str", 200), + "膜片": ("str", 200), + "膜片材质": ("str", 200), + "CRM明细ID号": ("str", 200), + "成品物料编码": ("str", 200), + "工单号": ("str", 200), + "盘号": ("str", 200), + "编码": ("str", 200), + "型批名称": ("str", 200), + "型批型号": ("str", 200), + "开票名称": ("str", 200), + "开票型号": ("str", 200), + "上数据时间": ("date", None), + "生产代码1": ("str", 50), + "生产代码2": ("str", 50) +} + +CONTRACT_DATA_FIELDS = { + "合同年份": ("str", 10), + "车间号": ("str", 20), + "工令号": ("str", 200), + "订单号": ("str", 150), + "客户名称": ("str", 200), + "产品型号": ("str", 200), + "量程": ("str", 150), + "数量": ("int", None), + "单价": ("int", None), + "ID": ("int", None), + "位号": ("str", 500) +} + +CONTRACT_DATA_MAPPING = { + "合同年份": "合同年份", + "车间号": "车间号", + "工令号": "工令号", + "订单号": "订单号", + "客户名称": "客户名称", + "产品型号": "选型型号", + "量程": "量程", + "数量": "数量", + "单价": None, + "ID": None, + "位号": "位号" +} diff --git a/config/database.py b/config/database.py new file mode 100644 index 0000000..e24e4d8 --- /dev/null +++ b/config/database.py @@ -0,0 +1,30 @@ +# config/database.py +# 统一的数据库配置 + +# ================= SQL Server 配置 ================= +SQL_SERVER_CONFIG = { + 'driver': 'ODBC Driver 18 for SQL Server', + 'server': '192.168.110.114', + 'database': 'CompanyDB', + 'username': 'peng', + 'password': 'Cqbld123456.', + 'trust_server_certificate': 'yes' +} + +# ================= Access 驱动配置 ================= +ACCESS_DRIVER = "{Microsoft Access Driver (*.mdb, *.accdb)}" + +# ================= 兼容性别名 ================= +# 为了兼容旧代码，提供不同命名风格的别名 + +# SQL_SERVER_CONN - 兼容 db_utils.py (使用 uid/pwd) +SQL_SERVER_CONN = { + 'driver': f"{{{SQL_SERVER_CONFIG['driver']}}}", + 'server': SQL_SERVER_CONFIG['server'], + 'database': SQL_SERVER_CONFIG['database'], + 'uid': SQL_SERVER_CONFIG['username'], + 'pwd': SQL_SERVER_CONFIG['password'] +} + +# DB_CONFIG - 兼容 etl_manager.py, migration.py (使用 username/password) +DB_CONFIG = SQL_SERVER_CONFIG.copy() diff --git a/config/field_mappings.py b/config/field_mappings.py new file mode 100644 index 0000000..ee20a64 --- /dev/null +++ b/config/field_mappings.py @@ -0,0 +1,73 @@ +# config/field_mappings.py +# 字段映射和清洗规则配置 + +import os +from config.app_settings import EXECUTION_CARD_FIELDS + +# ================= 字段清洗规则 ================= +# 从原 update_config.py 抽取 +# 基于 NVARCHAR (按字符数计算长度) +TABLE_SCHEMA = { + "合同年份": {"type": "str", "max_len": 10}, + "总排号": {"type": "str", "max_len": 50}, + "序号": {"type": "int"}, + "订单号": {"type": "str", "max_len": 150}, + "车间号": {"type": "str", "max_len": 20}, + "销售内部号": {"type": "str", "max_len": 50}, + "经办人": {"type": "str", "max_len": 20}, + "签订日期": {"type": "date"}, + "交货日期": {"type": "date"}, + "客户名称": {"type": "str", "max_len": 200}, + "产品名称": {"type": "str", "max_len": 200}, + "客户型号": {"type": "str", "max_len": 200}, + "选型型号": {"type": "str", "max_len": 200}, + "量程": {"type": "str", "max_len": 150}, + "数量": {"type": "int"}, + "备注2": {"type": "str", "max_len": 500}, + "备注1": {"type": "str", "max_len": 500}, + "位号": {"type": "str", "max_len": 500}, + "技术参数": {"type": "str", "max_len": 1000}, + "车间": {"type": "str", "max_len": 200}, + "工令号": {"type": "str", "max_len": 200}, + "接单日期": {"type": "date"}, + "新参数": {"type": "str", "max_len": 1000}, + "基本型号": {"type": "str", "max_len": 200}, + "公称外径": {"type": "str", "max_len": 200}, + "安装代码": {"type": "str", "max_len": 200}, + "设计形式": {"type": "str", "max_len": 200}, + "技术安装代码": {"type": "str", "max_len": 200}, + "隔膜类型": {"type": "str", "max_len": 200}, + "标准": {"type": "str", "max_len": 100}, + "隔膜大小": {"type": "str", "max_len": 200}, + "隔膜材质": {"type": "str", "max_len": 200}, + "膜片": {"type": "str", "max_len": 200}, + "膜片材质": {"type": "str", "max_len": 200}, + "CRM明细ID号": {"type": "str", "max_len": 200}, + "成品物料编码": {"type": "str", "max_len": 200}, + "工单号": {"type": "str", "max_len": 200}, + "盘号": {"type": "str", "max_len": 200}, + "编码": {"type": "str", "max_len": 200}, + "型批名称": {"type": "str", "max_len": 200}, + "型批型号": {"type": "str", "max_len": 200}, + "开票名称": {"type": "str", "max_len": 200}, + "开票型号": {"type": "str", "max_len": 200}, + "上数据时间": {"type": "date"}, + "生产代码1": {"type": "str", "max_len": 50}, + "生产代码2": {"type": "str", "max_len": 50} +} + +# ================= 合同数据映射 ================= +# 从原 update_config.py 抽取 +CONTRACT_MAPPING = { + "合同年份": "合同年份", + "车间号": "车间号", + "工令号": "工令号", + "订单号": "订单号", + "客户名称": "客户名称", + "产品型号": "选型型号", + "量程": "量程", + "数量": "数量", + "单价": None, + "ID": None, + "位号": "位号" +} diff --git a/config.py b/config/file_sources.py similarity index 50% rename from config.py rename to config/file_sources.py index 9d61206..73a417d 100644 --- a/config.py +++ b/config/file_sources.py @@ -1,39 +1,15 @@ -# config.py +# config/file_sources.py +# 文件路径配置 - 从原 config.py 和 update_config.py 抽离 -# ================= 数据库连接配置 ================= -SQL_SERVER_CONN = { - 'driver': '{ODBC Driver 18 for SQL Server}', - 'server': '192.168.110.114', - 'database': 'CompanyDB', - 'uid': 'peng', - 'pwd': 'Cqbld123456.' -} - -# Access 驱动配置 (根据你的 Office 版本选择 mdb 或 accdb) -ACCESS_DRIVER = "{Microsoft Access Driver (*.mdb, *.accdb)}" - -# ================= 日志表配置 ================= -# 根据你提供的表结构配置列名 -LOG_TABLE_CONFIG = { - 'schema': 'dbo', - 'table_name': 'TableChangeLog', - # 下面定义字段名，方便代码引用，防止硬编码 - 'col_log_id': 'LogID', - 'col_table_name': 'TableName', - 'col_record_id': 'RecordID', - 'col_address': 'TableAddress', # 使用 TableAddress 作为文件路径来源 - 'col_synced': 'Synced' -} - -# ================= 同步映射配置 (核心重构) ================= +# ================= Access 同步映射配置 ================= +# 从原 config.py 抽取 # 结构： { "Access文件绝对路径": { "Access表名": { SQL目标配置 } } } -# 这样即使不同文件有相同的表名，也能区分开 SYNC_MAPPING = { # 第一个 Access 文件 r"\\192.168.110.114\生产进度表\2025年数据\生产合同数据.accdb": { "26年压力表合同数据": { "target_schema": "productionContractData", - "target_table": "26年压力表合同数据", # 映射到 A 表 + "target_table": "26年压力表合同数据", "pk_col": "ID" }, "26年温度计合同数据": { @@ -51,289 +27,424 @@ SYNC_MAPPING = { "target_table": "25年温度计合同数据", "pk_col": "ID" } - }, r"\\192.168.110.114\生产进度表\2026年数据\成品入库.accdb": { "成品交检记录": { "target_schema": "productWarehousing", - "target_table": "成品交检记录_YEAR2026", + "target_table": "成品交检记录_YEAR2026", "pk_col": "ID" }, "成品入库记录": { "target_schema": "productWarehousing", - "target_table": "成品入库记录_YEAR2026", + "target_table": "成品入库记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\一车间.accdb": { "一车间记录": { "target_schema": "workshopOne", - "target_table": "一车间记录_YEAR2026", + "target_table": "一车间记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\二车间.accdb": { "二车间记录": { "target_schema": "workshopTwo", - "target_table": "二车间记录_YEAR2026", + "target_table": "二车间记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\三车间.accdb": { "三车间记录": { "target_schema": "workshopThree", - "target_table": "三车间记录_YEAR2026", + "target_table": "三车间记录_YEAR2026", "pk_col": "ID" }, "温度计调校记录": { "target_schema": "workshopThree", - "target_table": "温度计调校记录_YEAR2026", + "target_table": "温度计调校记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\机加工.accdb": { "隔膜机加接收": { "target_schema": "machining", - "target_table": "隔膜机加接收_YEAR2026", + "target_table": "隔膜机加接收_YEAR2026", "pk_col": "ID" }, "膜片焊接": { "target_schema": "machining", - "target_table": "膜片焊接_YEAR2026", + "target_table": "膜片焊接_YEAR2026", "pk_col": "ID" }, "膜片接收": { "target_schema": "machining", - "target_table": "膜片接收_YEAR2026", + "target_table": "膜片接收_YEAR2026", "pk_col": "ID" }, "喷涂寄出": { "target_schema": "machining", - "target_table": "喷涂寄出_YEAR2026", + "target_table": "喷涂寄出_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\计划.accdb": { "报警器接单记录": { "target_schema": "contractPlanning", - "target_table": "报警器接单记录_YEAR2026", + "target_table": "报警器接单记录_YEAR2026", "pk_col": "ID" }, "二车间接单记录": { "target_schema": "contractPlanning", - "target_table": "二车间接单记录_YEAR2026", + "target_table": "二车间接单记录_YEAR2026", "pk_col": "ID" }, "隔膜接单记录": { "target_schema": "contractPlanning", - "target_table": "隔膜接单记录_YEAR2026", + "target_table": "隔膜接单记录_YEAR2026", "pk_col": "ID" }, "过压保护器接单记录": { "target_schema": "contractPlanning", - "target_table": "过压保护器接单记录_YEAR2026", + "target_table": "过压保护器接单记录_YEAR2026", "pk_col": "ID" }, "精密表部件接收记录": { "target_schema": "contractPlanning", - "target_table": "精密表部件接收记录_YEAR2026", + "target_table": "精密表部件接收记录_YEAR2026", "pk_col": "ID" }, "精密表接单记录": { "target_schema": "contractPlanning", - "target_table": "精密表接单记录_YEAR2026", + "target_table": "精密表接单记录_YEAR2026", "pk_col": "ID" }, "库房接单记录": { "target_schema": "contractPlanning", - "target_table": "库房接单记录_YEAR2026", + "target_table": "库房接单记录_YEAR2026", "pk_col": "ID" }, "软管接单记录": { "target_schema": "contractPlanning", - "target_table": "软管接单记录_YEAR2026", + "target_table": "软管接单记录_YEAR2026", "pk_col": "ID" }, "氩弧焊接单记录": { "target_schema": "contractPlanning", - "target_table": "氩弧焊接单记录_YEAR2026", + "target_table": "氩弧焊接单记录_YEAR2026", "pk_col": "ID" }, "一车间接单记录": { "target_schema": "contractPlanning", - "target_table": "一车间接单记录_YEAR2026", + "target_table": "一车间接单记录_YEAR2026", "pk_col": "ID" }, "阻尼器接单记录": { "target_schema": "contractPlanning", - "target_table": "阻尼器接单记录_YEAR2026", + "target_table": "阻尼器接单记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\检验记录数据库.accdb": { "检验合格记录表": { "target_schema": "inspectionRecords", - "target_table": "检验合格记录表_YEAR2026", + "target_table": "检验合格记录表_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\零件库.accdb": { "表盘缺件记录": { "target_schema": "partsWarehouse", - "target_table": "表盘缺件记录_YEAR2026", + "target_table": "表盘缺件记录_YEAR2026", "pk_col": "ID" }, "表盘入库记录": { "target_schema": "partsWarehouse", - "target_table": "表盘入库记录_YEAR2026", + "target_table": "表盘入库记录_YEAR2026", "pk_col": "ID" }, "部件发放记录": { "target_schema": "partsWarehouse", - "target_table": "部件发放记录_YEAR2026", + "target_table": "部件发放记录_YEAR2026", "pk_col": "ID" }, "出库单发出记录": { "target_schema": "partsWarehouse", - "target_table": "出库单发出记录_YEAR2026", + "target_table": "出库单发出记录_YEAR2026", "pk_col": "ID" }, "出库单记录": { "target_schema": "partsWarehouse", - "target_table": "出库单记录_YEAR2026", + "target_table": "出库单记录_YEAR2026", "pk_col": "ID" }, "法兰缺件记录": { "target_schema": "partsWarehouse", - "target_table": "法兰缺件记录_YEAR2026", + "target_table": "法兰缺件记录_YEAR2026", "pk_col": "ID" }, "法兰入库记录": { "target_schema": "partsWarehouse", - "target_table": "法兰入库记录_YEAR2026", + "target_table": "法兰入库记录_YEAR2026", "pk_col": "ID" }, "卡到库房记录": { "target_schema": "partsWarehouse", - "target_table": "卡到库房记录_YEAR2026", + "target_table": "卡到库房记录_YEAR2026", "pk_col": "ID" }, "喷涂到厂": { "target_schema": "partsWarehouse", - "target_table": "喷涂到厂_YEAR2026", + "target_table": "喷涂到厂_YEAR2026", "pk_col": "ID" }, "缺件记录": { "target_schema": "partsWarehouse", - "target_table": "缺件记录_YEAR2026", + "target_table": "缺件记录_YEAR2026", "pk_col": "ID" }, "温度计法兰缺件记录": { "target_schema": "partsWarehouse", - "target_table": "温度计法兰缺件记录_YEAR2026", + "target_table": "温度计法兰缺件记录_YEAR2026", "pk_col": "ID" }, "温度计法兰入库记录": { "target_schema": "partsWarehouse", - "target_table": "温度计法兰入库记录_YEAR2026", + "target_table": "温度计法兰入库记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\温度计记录.accdb": { "采 技 机记录": { "target_schema": "thermometerRecord", - "target_table": "采 技 机记录_YEAR2026", + "target_table": "采 技 机记录_YEAR2026", "pk_col": "ID" }, "温度计调校记录": { "target_schema": "thermometerRecord", - "target_table": "温度计调校记录_YEAR2026", + "target_table": "温度计调校记录_YEAR2026", "pk_col": "ID" }, "温度计检验记录": { "target_schema": "thermometerRecord", - "target_table": "温度计检验记录_YEAR2026", + "target_table": "温度计检验记录_YEAR2026", "pk_col": "ID" }, "温度计组装记录": { "target_schema": "thermometerRecord", - "target_table": "温度计组装记录_YEAR2026", + "target_table": "温度计组装记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\锡焊数据.accdb": { "操作者1完成记录": { "target_schema": "solderingData", - "target_table": "操作者1完成记录_YEAR2026", + "target_table": "操作者1完成记录_YEAR2026", "pk_col": "ID" }, "操作者2完成记录": { "target_schema": "solderingData", - "target_table": "操作者2完成记录_YEAR2026", + "target_table": "操作者2完成记录_YEAR2026", "pk_col": "ID" }, "零件到车间记录": { "target_schema": "solderingData", - "target_table": "零件到车间记录_YEAR2026", + "target_table": "零件到车间记录_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\氩弧焊.accdb": { "表壳焊接记录": { "target_schema": "TIGWelding", - "target_table": "表壳焊接记录_YEAR2026", + "target_table": "表壳焊接记录_YEAR2026", "pk_col": "ID" }, "超压": { "target_schema": "TIGWelding", - "target_table": "超压_YEAR2026", + "target_table": "超压_YEAR2026", "pk_col": "ID" }, "氦测": { "target_schema": "TIGWelding", - "target_table": "氦测_YEAR2026", + "target_table": "氦测_YEAR2026", "pk_col": "ID" }, "接收": { "target_schema": "TIGWelding", - "target_table": "接收_YEAR2026", + "target_table": "接收_YEAR2026", "pk_col": "ID" }, "退火": { "target_schema": "TIGWelding", - "target_table": "退火_YEAR2026", + "target_table": "退火_YEAR2026", "pk_col": "ID" }, "氩弧焊记录": { "target_schema": "TIGWelding", - "target_table": "氩弧焊记录_YEAR2026", + "target_table": "氩弧焊记录_YEAR2026", "pk_col": "ID" }, "氩弧焊领料": { "target_schema": "TIGWelding", - "target_table": "氩弧焊领料_YEAR2026", + "target_table": "氩弧焊领料_YEAR2026", "pk_col": "ID" } }, r"\\192.168.110.114\生产进度表\2026年数据\执行卡下发记录.accdb": { "执行卡下发记录": { "target_schema": "executionCardIssuanceRecord", - "target_table": "执行卡下发记录_YEAR2026", + "target_table": "执行卡下发记录_YEAR2026", "pk_col": "ID" } } } -# ================= ntfy 配置 ================= -NTFY_CONFIG = { - 'enabled': True, # 是否启用通知 - 'server_url': 'https://ntfy.server10086.icu', # 如果是自建服务器，请修改为自己的 URL - 'topic': 'bld', # 你的订阅主题 - 'token': 'tk_eop5fs66acxtwxf6vlkiojhdvkgb0', # <--- 在这里填入你的 Access Token - 'priority': { - 'error': 'high', # 错误消息优先级 - 'critical': 'urgent' # 严重错误优先级 + +# ================= Excel 文件配置 ================= +# 从原 update_config.py 抽取 +EXCEL_CONFIGS = [ + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm", + "sheet_names": ["Sheet1"], + "contract_year": "2022", + "field_mapping": { + "产品型号": "选型型号", + "备注": "备注1", + "下单日期": "接单日期" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm", + "sheet_names": ["Sheet1"], + "contract_year": "2023", + "field_mapping": { + "产品型号": "选型型号", + "备注": "备注1", + "下单日期": "接单日期" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm", + "sheet_names": ["Sheet1"], + "contract_year": "2023", + "field_mapping": { + "产品型号": "选型型号", + "备注": "备注1", + "下单日期": "接单日期" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm", + "sheet_names": ["重庆数据","北京数据"], + "contract_year": "2024", + "field_mapping": { + "产品型号": "选型型号", + "备注": "备注1", + "下单日期": "接单日期" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm", + "sheet_names": ["重庆数据","北京数据"], + "contract_year": "2024", + "field_mapping": { + "产品型号": "选型型号", + "备注": "备注1", + "下单日期": "接单日期" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm", + "sheet_names": ["重庆数据","北京数据"], + "contract_year": "2025", + "field_mapping": { + "产品型号": "选型型号", + "备注": "备注1", + "下单日期": "接单日期" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm", + "sheet_names": ["重庆数据","北京数据"], + "contract_year": "2026", + "field_mapping": { + "产品型号": "选型型号", + "备注": "备注1", + "下单日期": "接单日期" + } } -} -# ================= 运行参数 ================= -POLL_INTERVAL = 5 # 轮询间隔(秒) -BATCH_SIZE = 10000 # 批量处理大小 \ No newline at end of file +] + +# ================= 迁移任务配置 ================= +# 从原 migration.py 抽取 +MIGRATION_TASKS = [ + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm", + "year": 2022, + "sheet_names": ["Sheet1"], + "mapping": { + "车间号": "车间号", + "工令号": "工令号", + "客户型号": "客户型号" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm", + "year": 2023, + "sheet_names": ["Sheet1"], + "mapping": { + "车间号": "车间号", + "工令号": "工令号", + "客户型号": "客户型号" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm", + "year": 2023, + "sheet_names": ["Sheet1"], + "mapping": { + "车间号": "车间号", + "工令号": "工令号", + "客户型号": "客户型号" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm", + "year": 2024, + "sheet_names": ["重庆数据","北京数据"], + "mapping": { + "车间号": "车间号", + "工令号": "工令号", + "客户型号": "客户型号" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm", + "year": 2024, + "sheet_names": ["重庆数据","北京数据"], + "mapping": { + "车间号": "车间号", + "工令号": "工令号", + "客户型号": "客户型号" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm", + "year": 2025, + "sheet_names": ["重庆数据","北京数据"], + "mapping": { + "车间号": "车间号", + "工令号": "工令号", + "客户型号": "客户型号" + } + }, + { + "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm", + "year": 2026, + "sheet_names": ["重庆数据","北京数据"], + "mapping": { + "车间号": "车间号", + "工令号": "工令号", + "客户型号": "客户型号" + } + } +] diff --git a/db_utils.py b/db_utils.py index c6d15eb..fa89532 100644 --- a/db_utils.py +++ b/db_utils.py @@ -1,21 +1,20 @@ # db_utils.py import pyodbc -import config +from config import SQL_SERVER_CONN, ACCESS_DRIVER def get_sql_conn(): """获取 SQL Server 连接""" - c = config.SQL_SERVER_CONN # 显式添加 TrustServerCertificate=yes 以兼容 ODBC Driver 18+ conn_str = ( - f"DRIVER={c['driver']};SERVER={c['server']};" - f"DATABASE={c['database']};UID={c['uid']};PWD={c['pwd']};" - "Encrypt=yes;TrustServerCertificate=yes;" + f"DRIVER={SQL_SERVER_CONN['driver']};SERVER={SQL_SERVER_CONN['server']};" + f"DATABASE={SQL_SERVER_CONN['database']};UID={SQL_SERVER_CONN['uid']};PWD={SQL_SERVER_CONN['pwd']};" + "Encrypt=yes;TrustServerCertificate=yes;" ) return pyodbc.connect(conn_str) def get_access_conn(file_path): """获取 Access 连接""" - conn_str = f"DRIVER={config.ACCESS_DRIVER};DBQ={file_path};" + conn_str = f"DRIVER={ACCESS_DRIVER};DBQ={file_path};" return pyodbc.connect(conn_str) def fmt_table(schema, table): diff --git a/etl_manager.py b/etl_manager.py index 26e8e6b..6ac3493 100644 --- a/etl_manager.py +++ b/etl_manager.py @@ -13,7 +13,7 @@ from sqlalchemy.engine import URL from sqlalchemy.types import NVARCHAR, Integer, Date # 导入配置 -import update_config as config +from config import DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA # ================= 抑制 openpyxl 的数据验证警告 ================= warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl') @@ -37,19 +37,19 @@ class DataSynchronizer: def __init__(self, force_sync=False): self.force_sync = force_sync self.engine = self._get_db_connection() - self.cache_dir = config.CACHE_DIR + self.cache_dir = CACHE_DIR if not os.path.exists(self.cache_dir): os.makedirs(self.cache_dir) def _get_db_connection(self): connection_string = ( - f"DRIVER={{{config.DB_CONFIG['driver']}}};" - f"SERVER={config.DB_CONFIG['server']};" - f"DATABASE={config.DB_CONFIG['database']};" - f"UID={config.DB_CONFIG['username']};" - f"PWD={config.DB_CONFIG['password']};" - f"TrustServerCertificate={config.DB_CONFIG.get('TrustServerCertificate', 'no')};" + f"DRIVER={{{DB_CONFIG['driver']}}};" + f"SERVER={DB_CONFIG['server']};" + f"DATABASE={DB_CONFIG['database']};" + f"UID={DB_CONFIG['username']};" + f"PWD={DB_CONFIG['password']};" + f"TrustServerCertificate={DB_CONFIG.get('TrustServerCertificate', 'no')};" ) connection_url = URL.create("mssql+pyodbc", query={"odbc_connect": connection_string}) return create_engine(connection_url, fast_executemany=True) @@ -94,7 +94,7 @@ class DataSynchronizer: df = df.drop_duplicates(subset=['总排号'], keep='first') # 3. 补全列 - for col in config.TABLE_SCHEMA.keys(): + for col in TABLE_SCHEMA.keys(): if col not in df.columns: df[col] = None @@ -102,7 +102,7 @@ class DataSynchronizer: dtype_dict = {} # 4. 字段清洗 - for col, rules in config.TABLE_SCHEMA.items(): + for col, rules in TABLE_SCHEMA.items(): if col not in df.columns: continue @@ -136,7 +136,7 @@ class DataSynchronizer: dtype_dict[col] = NVARCHAR(max_len) - final_cols = list(config.TABLE_SCHEMA.keys()) + final_cols = list(TABLE_SCHEMA.keys()) return df[final_cols], dtype_dict @@ -162,7 +162,7 @@ class DataSynchronizer: if not df_insert.empty: logger.info("正在执行批量插入...") df_insert.to_sql('executionCardData', self.engine, schema='warehouseOutbound', - if_exists='append', index=False, chunksize=config.BATCH_SIZE, + if_exists='append', index=False, chunksize=BATCH_SIZE, dtype=dtype_dict) logger.info("批量插入完成。") @@ -198,7 +198,7 @@ class DataSynchronizer: logger.info(f"批量更新完成，共影响 {affected_rows} 行。") def process_excel_files(self): - for cfg in config.EXCEL_CONFIGS: + for cfg in EXCEL_CONFIGS: remote_path = cfg['file_path'] filename = os.path.basename(remote_path) local_path = os.path.join(self.cache_dir, filename) diff --git a/init_full_sync.py b/init_full_sync.py index b84f4d5..0e6c334 100644 --- a/init_full_sync.py +++ b/init_full_sync.py @@ -1,5 +1,5 @@ import pyodbc -import config +from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, BATCH_SIZE import db_utils import time import os @@ -103,8 +103,8 @@ def run_full_sync(): logger.info("=" * 70) log_start(logger, "全量初始化同步") - log_info(logger, f"配置文件数: {len(config.SYNC_MAPPING)} 个") - log_info(logger, f"批次大小: {config.BATCH_SIZE} 行") + log_info(logger, f"配置文件数: {len(SYNC_MAPPING)} 个") + log_info(logger, f"批次大小: {BATCH_SIZE} 行") logger.info("=" * 70) sync_start_time = time.time() @@ -123,7 +123,7 @@ def run_full_sync(): return # 第一层循环:遍历配置文件中的所有文件路径 - for acc_path, tables_map in config.SYNC_MAPPING.items(): + for acc_path, tables_map in SYNC_MAPPING.items(): if not os.path.exists(acc_path): log_skip(logger, f"文件不存在: {acc_path}") continue @@ -184,7 +184,7 @@ def run_full_sync(): last_log_time = start_time while True: - rows = acc_cursor.fetchmany(config.BATCH_SIZE) + rows = acc_cursor.fetchmany(BATCH_SIZE) if not rows: break sql_cursor.executemany(insert_sql, rows) diff --git a/migration.py b/migration.py index f0de989..66def12 100644 --- a/migration.py +++ b/migration.py @@ -3,21 +3,13 @@ import os import shutil import urllib from sqlalchemy import create_engine, text +from config import DB_CONFIG, MIGRATION_TASKS, TEMP_DIR import ntfy_utils # 确保该文件在同一目录下 # ========================================== # 1. 脚本配置 (Configuration) # ========================================== -# 数据库连接信息 -DB_CONFIG = { - "server": "192.168.110.114", - "database": "CompanyDB", - "username": "peng", - "password": "Cqbld123456.", - "driver": "ODBC Driver 18 for SQL Server" -} - # 目标表配置 TARGET_DB_SCHEMA = "warehouseOutbound" TARGET_TABLE_NAME = "customerProductType" @@ -31,81 +23,6 @@ SQL_COL_MODEL = "客户型号" # 运行参数 FORCE_UPDATE = False # 如果设为 True，则无视时间对比，强制更新所有文件 -TEMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "temp") - -# 迁移任务清单 -MIGRATION_TASKS = [ - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm", - "year": 2022, - "sheet_names": ["Sheet1"], - "mapping": { - "车间号": SQL_COL_WORKSHOP, - "工令号": SQL_COL_ORDER, - "客户型号": SQL_COL_MODEL - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm", - "year": 2023, - "sheet_names": ["Sheet1"], - "mapping": { - "车间号": SQL_COL_WORKSHOP, - "工令号": SQL_COL_ORDER, - "客户型号": SQL_COL_MODEL - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm", - "year": 2023, - "sheet_names": ["Sheet1"], - "mapping": { - "车间号": SQL_COL_WORKSHOP, - "工令号": SQL_COL_ORDER, - "客户型号": SQL_COL_MODEL - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm", - "year": 2024, - "sheet_names": ["重庆数据","北京数据"], - "mapping": { - "车间号": SQL_COL_WORKSHOP, - "工令号": SQL_COL_ORDER, - "客户型号": SQL_COL_MODEL - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm", - "year": 2024, - "sheet_names": ["重庆数据","北京数据"], - "mapping": { - "车间号": SQL_COL_WORKSHOP, - "工令号": SQL_COL_ORDER, - "客户型号": SQL_COL_MODEL - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm", - "year": 2025, - "sheet_names": ["重庆数据","北京数据"], - "mapping": { - "车间号": SQL_COL_WORKSHOP, - "工令号": SQL_COL_ORDER, - "客户型号": SQL_COL_MODEL - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm", - "year": 2026, - "sheet_names": ["重庆数据","北京数据"], - "mapping": { - "车间号": SQL_COL_WORKSHOP, - "工令号": SQL_COL_ORDER, - "客户型号": SQL_COL_MODEL - } - } -] # ========================================== # 2. 核心辅助函数 diff --git a/ntfy_utils.py b/ntfy_utils.py index 71a5e06..35ce55c 100644 --- a/ntfy_utils.py +++ b/ntfy_utils.py @@ -1,12 +1,12 @@ # ntfy_utils.py import requests -import config +from config import NTFY_CONFIG def send_ntfy(message, title="数据库同步消息", priority="default", tags=None): """ 向加密的 ntfy 服务器发送消息 """ - conf = config.NTFY_CONFIG + conf = NTFY_CONFIG if not conf.get('enabled', False): return @@ -51,7 +51,7 @@ def send_error(msg): send_ntfy( message=str(msg), title="❌ 同步任务错误", - priority=config.NTFY_CONFIG['priority']['error'], + priority=NTFY_CONFIG['priority']['error'], tags=["warning", "database"] ) @@ -60,6 +60,6 @@ def send_critical(msg): send_ntfy( message=str(msg), title="🔥 同步服务崩溃", - priority=config.NTFY_CONFIG['priority']['critical'], + priority=NTFY_CONFIG['priority']['critical'], tags=["skull", "critical"] ) \ No newline at end of file diff --git a/run_incremental_sync.py b/run_incremental_sync.py index cbcaa51..0c3405e 100644 --- a/run_incremental_sync.py +++ b/run_incremental_sync.py @@ -2,7 +2,7 @@ import time import os import sys import pyodbc -import config +from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, POLL_INTERVAL, LOG_TABLE_CONFIG import db_utils import logging from logging.handlers import TimedRotatingFileHandler @@ -95,7 +95,7 @@ def log_sync(message): def process_sync_task(): """ 逻辑重构： - 遍历 config.SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。 + 遍历 SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。 """ sql_conn = db_utils.get_sql_conn() sql_cursor = sql_conn.cursor() @@ -104,12 +104,12 @@ def process_sync_task(): # 标记是否有工作被处理（用于控制轮询休眠时间） work_done = False - cols = config.LOG_TABLE_CONFIG + cols = LOG_TABLE_CONFIG log_full_name = db_utils.fmt_table(cols['schema'], cols['table_name']) try: # === 核心循环：以配置文件为驱动 === - for clean_path, tables_map in config.SYNC_MAPPING.items(): + for clean_path, tables_map in SYNC_MAPPING.items(): # 1. 准备查询条件 # 获取该文件下所有需要同步的表名列表 @@ -265,8 +265,8 @@ def process_sync_task(): if __name__ == "__main__": log_start("增量同步服务已启动 (配置驱动模式)") - log_info(f"轮询间隔: {config.POLL_INTERVAL} 秒") - log_info(f"监控配置: {len(config.SYNC_MAPPING)} 个文件") + log_info(f"轮询间隔: {POLL_INTERVAL} 秒") + log_info(f"监控配置: {len(SYNC_MAPPING)} 个文件") logger.info("=" * 70) while True: @@ -274,7 +274,7 @@ if __name__ == "__main__": has_work = process_sync_task() # 如果有工作，说明可能还有积压，休息短一点(0.1s) # 如果没工作，休息标准间隔(5s) - time.sleep(0.1 if has_work else config.POLL_INTERVAL) + time.sleep(0.1 if has_work else POLL_INTERVAL) except KeyboardInterrupt: logger.info("=" * 70) log_stop("收到停止信号，服务正在关闭...") diff --git a/update_config.py b/update_config.py deleted file mode 100644 index 204a2fd..0000000 --- a/update_config.py +++ /dev/null @@ -1,154 +0,0 @@ -import os - -# ================= 数据库配置 ================= -DB_CONFIG = { - "server": "192.168.110.114", - "database": "CompanyDB", - "username": "peng", - "password": "Cqbld123456.", - "driver": "ODBC Driver 18 for SQL Server", - "TrustServerCertificate": "yes" -} - -# ================= 全局配置 ================= -CACHE_DIR = os.path.join(os.getcwd(), "temp") -BATCH_SIZE = 5000 - -# ================= 字段清洗规则 ================= -# 基于 NVARCHAR (按字符数计算长度) -TABLE_SCHEMA = { - "合同年份": {"type": "str", "max_len": 10}, - "总排号": {"type": "str", "max_len": 50}, - "序号": {"type": "int"}, - "订单号": {"type": "str", "max_len": 150}, - "车间号": {"type": "str", "max_len": 20}, - "销售内部号": {"type": "str", "max_len": 50}, - "经办人": {"type": "str", "max_len": 20}, - "签订日期": {"type": "date"}, - "交货日期": {"type": "date"}, - "客户名称": {"type": "str", "max_len": 200}, - "产品名称": {"type": "str", "max_len": 200}, - "客户型号": {"type": "str", "max_len": 200}, - "选型型号": {"type": "str", "max_len": 200}, - "量程": {"type": "str", "max_len": 150}, - "数量": {"type": "int"}, - "备注2": {"type": "str", "max_len": 500}, - "备注1": {"type": "str", "max_len": 500}, - "位号": {"type": "str", "max_len": 500}, - "技术参数": {"type": "str", "max_len": 1000}, - "车间": {"type": "str", "max_len": 200}, - "工令号": {"type": "str", "max_len": 200}, - "接单日期": {"type": "date"}, - "新参数": {"type": "str", "max_len": 1000}, - "基本型号": {"type": "str", "max_len": 200}, - "公称外径": {"type": "str", "max_len": 200}, - "安装代码": {"type": "str", "max_len": 200}, - "设计形式": {"type": "str", "max_len": 200}, - "技术安装代码": {"type": "str", "max_len": 200}, - "隔膜类型": {"type": "str", "max_len": 200}, - "标准": {"type": "str", "max_len": 100}, - "隔膜大小": {"type": "str", "max_len": 200}, - "隔膜材质": {"type": "str", "max_len": 200}, - "膜片": {"type": "str", "max_len": 200}, - "膜片材质": {"type": "str", "max_len": 200}, - "CRM明细ID号": {"type": "str", "max_len": 200}, - "成品物料编码": {"type": "str", "max_len": 200}, - "工单号": {"type": "str", "max_len": 200}, - "盘号": {"type": "str", "max_len": 200}, - "编码": {"type": "str", "max_len": 200}, - "型批名称": {"type": "str", "max_len": 200}, - "型批型号": {"type": "str", "max_len": 200}, - "开票名称": {"type": "str", "max_len": 200}, - "开票型号": {"type": "str", "max_len": 200}, - "上数据时间": {"type": "date"}, - "生产代码1": {"type": "str", "max_len": 50}, - "生产代码2": {"type": "str", "max_len": 50} -} - -CONTRACT_MAPPING = { - "合同年份": "合同年份", - "车间号": "车间号", - "工令号": "工令号", - "订单号": "订单号", - "客户名称": "客户名称", - "产品型号": "选型型号", - "量程": "量程", - "数量": "数量", - "单价": None, - "ID": None, - "位号": "位号" -} - -EXCEL_CONFIGS = [ - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm", - "sheet_names": ["Sheet1"], - "contract_year": "2022", - "field_mapping": { - "产品型号": "选型型号", - "备注": "备注1", - "下单日期": "接单日期" - - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm", - "sheet_names": ["Sheet1"], - "contract_year": "2023", - "field_mapping": { - "产品型号": "选型型号", - "备注": "备注1", - "下单日期": "接单日期" - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm", - "sheet_names": ["Sheet1"], - "contract_year": "2023", - "field_mapping": { - "产品型号": "选型型号", - "备注": "备注1", - "下单日期": "接单日期" - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm", - "sheet_names": ["重庆数据","北京数据"], - "contract_year": "2024", - "field_mapping": { - "产品型号": "选型型号", - "备注": "备注1", - "下单日期": "接单日期" - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm", - "sheet_names": ["重庆数据","北京数据"], - "contract_year": "2024", - "field_mapping": { - "产品型号": "选型型号", - "备注": "备注1", - "下单日期": "接单日期" - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm", - "sheet_names": ["重庆数据","北京数据"], - "contract_year": "2025", - "field_mapping": { - "产品型号": "选型型号", - "备注": "备注1", - "下单日期": "接单日期" - } - }, - { - "file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm", - "sheet_names": ["重庆数据","北京数据"], - "contract_year": "2026", - "field_mapping": { - "产品型号": "选型型号", - "备注": "备注1", - "下单日期": "接单日期" - } - } -] \ No newline at end of file