AllForCompany/BLD_sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor configuration management and remove deprecated files

Browse Source 
- Consolidated database, file source, and field mapping configurations into dedicated modules under the `config` directory.
- Removed hardcoded database connection details from `migration.py` and replaced them with imports from the new configuration structure.
- Updated `ntfy_utils.py` and `run_incremental_sync.py` to utilize the new configuration imports for cleaner code and better maintainability.
- Deleted `update_config.py` as its contents have been integrated into the new configuration files.
- Added a new `settings.local.json` for managing permissions related to script execution.
- Enhanced the structure of the migration tasks and Excel configurations for better organization and clarity.
This commit is contained in:
Misaka_Company
2026-01-12 12:49:51 +08:00
parent ee14de0435
commit 83c90f0161
13 changed files with 501 additions and 365 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
.claude/settings.local.json Normal file
View File

@@ -0,0 +1,13 @@
{
"permissions": {
"allow": [
"Bash(python etl_manager.py --help)",
"Bash(python run_incremental_sync.py)",
"Bash(python -c \"import run_incremental_sync; print\\(''run_incremental_sync.py: import OK''\\)\")",
"Bash(python -c \"import etl_manager; print\\(''etl_manager.py: import OK''\\)\")",
"Bash(python -c \"import init_full_sync; print\\(''init_full_sync.py: import OK''\\)\")",
"Bash(python -c \"import migration; print\\(''migration.py: import OK''\\)\")",
"Bash(python migration.py)"
]
}
}

32
config/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
# config/__init__.py
# 统一配置导出接口
# 数据库配置
from .database import SQL_SERVER_CONFIG, SQL_SERVER_CONN, DB_CONFIG, ACCESS_DRIVER
# 文件路径配置
from .file_sources import SYNC_MAPPING, EXCEL_CONFIGS, MIGRATION_TASKS
# 字段映射配置
from .field_mappings import TABLE_SCHEMA, CONTRACT_MAPPING
# 应用设置
from .app_settings import (
LOG_TABLE_CONFIG, NTFY_CONFIG,
POLL_INTERVAL, BATCH_SIZE, CACHE_DIR, TEMP_DIR,
EXECUTION_CARD_FIELDS, CONTRACT_DATA_FIELDS, CONTRACT_DATA_MAPPING
)
# 统一导出列表
__all__ = [
# Database
'SQL_SERVER_CONFIG', 'SQL_SERVER_CONN', 'DB_CONFIG', 'ACCESS_DRIVER',
# File Sources
'SYNC_MAPPING', 'EXCEL_CONFIGS', 'MIGRATION_TASKS',
# Field Mappings
'TABLE_SCHEMA', 'CONTRACT_MAPPING',
# App Settings
'LOG_TABLE_CONFIG', 'NTFY_CONFIG',
'POLL_INTERVAL', 'BATCH_SIZE', 'CACHE_DIR', 'TEMP_DIR',
'EXECUTION_CARD_FIELDS', 'CONTRACT_DATA_FIELDS', 'CONTRACT_DATA_MAPPING'
]

115
config/app_settings.py Normal file
View File

@@ -0,0 +1,115 @@
# config/app_settings.py
# 应用设置配置
import os
# ================= 日志表配置 =================
# 从原 config.py 抽取
LOG_TABLE_CONFIG = {
'schema': 'dbo',
'table_name': 'TableChangeLog',
'col_log_id': 'LogID',
'col_table_name': 'TableName',
'col_record_id': 'RecordID',
'col_address': 'TableAddress',
'col_synced': 'Synced'
}
# ================= ntfy 配置 =================
# 从原 config.py 抽取
NTFY_CONFIG = {
'enabled': True,
'server_url': 'https://ntfy.server10086.icu',
'topic': 'bld',
'token': 'tk_eop5fs66acxtwxf6vlkiojhdvkgb0',
'priority': {
'error': 'high',
'critical': 'urgent'
}
}
# ================= 运行参数 =================
# 合并原 config.py 和 update_config.py 的配置
POLL_INTERVAL = 5 # 轮询间隔(秒)
BATCH_SIZE = 10000 # 批量处理大小
CACHE_DIR = os.path.join(os.getcwd(), "temp") # Excel 缓存目录
TEMP_DIR = os.path.join(os.getcwd(), "temp") # 临时目录（兼容 migration.py）
# ================= 字段配置 =================
# 从原 update_config.py (sync_excel_to_sql.py) 抽取
EXECUTION_CARD_FIELDS = {
"合同年份": ("str", 10),
"总排号": ("str", 50),
"序号": ("int", None),
"订单号": ("str", 150),
"车间号": ("str", 20),
"销售内部号": ("str", 50),
"经办人": ("str", 20),
"签订日期": ("date", None),
"交货日期": ("date", None),
"客户名称": ("str", 200),
"产品名称": ("str", 200),
"客户型号": ("str", 200),
"选型型号": ("str", 200),
"量程": ("str", 150),
"数量": ("int", None),
"备注2": ("str", 500),
"备注1": ("str", 500),
"位号": ("str", 500),
"技术参数": ("str", 1000),
"车间": ("str", 200),
"工令号": ("str", 200),
"接单日期": ("date", None),
"新参数": ("str", 1000),
"基本型号": ("str", 200),
"公称外径": ("str", 200),
"安装代码": ("str", 200),
"设计形式": ("str", 200),
"技术安装代码": ("str", 200),
"隔膜类型": ("str", 200),
"标准": ("str", 100),
"隔膜大小": ("str", 200),
"隔膜材质": ("str", 200),
"膜片": ("str", 200),
"膜片材质": ("str", 200),
"CRM明细ID号": ("str", 200),
"成品物料编码": ("str", 200),
"工单号": ("str", 200),
"盘号": ("str", 200),
"编码": ("str", 200),
"型批名称": ("str", 200),
"型批型号": ("str", 200),
"开票名称": ("str", 200),
"开票型号": ("str", 200),
"上数据时间": ("date", None),
"生产代码1": ("str", 50),
"生产代码2": ("str", 50)
}
CONTRACT_DATA_FIELDS = {
"合同年份": ("str", 10),
"车间号": ("str", 20),
"工令号": ("str", 200),
"订单号": ("str", 150),
"客户名称": ("str", 200),
"产品型号": ("str", 200),
"量程": ("str", 150),
"数量": ("int", None),
"单价": ("int", None),
"ID": ("int", None),
"位号": ("str", 500)
}
CONTRACT_DATA_MAPPING = {
"合同年份": "合同年份",
"车间号": "车间号",
"工令号": "工令号",
"订单号": "订单号",
"客户名称": "客户名称",
"产品型号": "选型型号",
"量程": "量程",
"数量": "数量",
"单价": None,
"ID": None,
"位号": "位号"
}

30
config/database.py Normal file
View File

@@ -0,0 +1,30 @@
# config/database.py
# 统一的数据库配置
# ================= SQL Server 配置 =================
SQL_SERVER_CONFIG = {
'driver': 'ODBC Driver 18 for SQL Server',
'server': '192.168.110.114',
'database': 'CompanyDB',
'username': 'peng',
'password': 'Cqbld123456.',
'trust_server_certificate': 'yes'
}
# ================= Access 驱动配置 =================
ACCESS_DRIVER = "{Microsoft Access Driver (*.mdb, *.accdb)}"
# ================= 兼容性别名 =================
# 为了兼容旧代码，提供不同命名风格的别名
# SQL_SERVER_CONN - 兼容 db_utils.py (使用 uid/pwd)
SQL_SERVER_CONN = {
'driver': f"{{{SQL_SERVER_CONFIG['driver']}}}",
'server': SQL_SERVER_CONFIG['server'],
'database': SQL_SERVER_CONFIG['database'],
'uid': SQL_SERVER_CONFIG['username'],
'pwd': SQL_SERVER_CONFIG['password']
}
# DB_CONFIG - 兼容 etl_manager.py, migration.py (使用 username/password)
DB_CONFIG = SQL_SERVER_CONFIG.copy()

73
config/field_mappings.py Normal file
View File

@@ -0,0 +1,73 @@
# config/field_mappings.py
# 字段映射和清洗规则配置
import os
from config.app_settings import EXECUTION_CARD_FIELDS
# ================= 字段清洗规则 =================
# 从原 update_config.py 抽取
# 基于 NVARCHAR (按字符数计算长度)
TABLE_SCHEMA = {
"合同年份": {"type": "str", "max_len": 10},
"总排号": {"type": "str", "max_len": 50},
"序号": {"type": "int"},
"订单号": {"type": "str", "max_len": 150},
"车间号": {"type": "str", "max_len": 20},
"销售内部号": {"type": "str", "max_len": 50},
"经办人": {"type": "str", "max_len": 20},
"签订日期": {"type": "date"},
"交货日期": {"type": "date"},
"客户名称": {"type": "str", "max_len": 200},
"产品名称": {"type": "str", "max_len": 200},
"客户型号": {"type": "str", "max_len": 200},
"选型型号": {"type": "str", "max_len": 200},
"量程": {"type": "str", "max_len": 150},
"数量": {"type": "int"},
"备注2": {"type": "str", "max_len": 500},
"备注1": {"type": "str", "max_len": 500},
"位号": {"type": "str", "max_len": 500},
"技术参数": {"type": "str", "max_len": 1000},
"车间": {"type": "str", "max_len": 200},
"工令号": {"type": "str", "max_len": 200},
"接单日期": {"type": "date"},
"新参数": {"type": "str", "max_len": 1000},
"基本型号": {"type": "str", "max_len": 200},
"公称外径": {"type": "str", "max_len": 200},
"安装代码": {"type": "str", "max_len": 200},
"设计形式": {"type": "str", "max_len": 200},
"技术安装代码": {"type": "str", "max_len": 200},
"隔膜类型": {"type": "str", "max_len": 200},
"标准": {"type": "str", "max_len": 100},
"隔膜大小": {"type": "str", "max_len": 200},
"隔膜材质": {"type": "str", "max_len": 200},
"膜片": {"type": "str", "max_len": 200},
"膜片材质": {"type": "str", "max_len": 200},
"CRM明细ID号": {"type": "str", "max_len": 200},
"成品物料编码": {"type": "str", "max_len": 200},
"工单号": {"type": "str", "max_len": 200},
"盘号": {"type": "str", "max_len": 200},
"编码": {"type": "str", "max_len": 200},
"型批名称": {"type": "str", "max_len": 200},
"型批型号": {"type": "str", "max_len": 200},
"开票名称": {"type": "str", "max_len": 200},
"开票型号": {"type": "str", "max_len": 200},
"上数据时间": {"type": "date"},
"生产代码1": {"type": "str", "max_len": 50},
"生产代码2": {"type": "str", "max_len": 50}
}
# ================= 合同数据映射 =================
# 从原 update_config.py 抽取
CONTRACT_MAPPING = {
"合同年份": "合同年份",
"车间号": "车间号",
"工令号": "工令号",
"订单号": "订单号",
"客户名称": "客户名称",
"产品型号": "选型型号",
"量程": "量程",
"数量": "数量",
"单价": None,
"ID": None,
"位号": "位号"
}

195
config.py → config/file_sources.py
View File

@@ -1,39 +1,15 @@
# config.py
# config/file_sources.py
# 文件路径配置 - 从原 config.py 和 update_config.py 抽离
# ================= 数据库连接配置 =================
SQL_SERVER_CONN = {
'driver': '{ODBC Driver 18 for SQL Server}',
'server': '192.168.110.114',
'database': 'CompanyDB',
'uid': 'peng',
'pwd': 'Cqbld123456.'
}
# Access 驱动配置 (根据你的 Office 版本选择 mdb 或 accdb)
ACCESS_DRIVER = "{Microsoft Access Driver (*.mdb, *.accdb)}"
# ================= 日志表配置 =================
# 根据你提供的表结构配置列名
LOG_TABLE_CONFIG = {
'schema': 'dbo',
'table_name': 'TableChangeLog',
# 下面定义字段名，方便代码引用，防止硬编码
'col_log_id': 'LogID',
'col_table_name': 'TableName',
'col_record_id': 'RecordID',
'col_address': 'TableAddress', # 使用 TableAddress 作为文件路径来源
'col_synced': 'Synced'
}
# ================= 同步映射配置 (核心重构) =================
# ================= Access 同步映射配置 =================
# 从原 config.py 抽取
# 结构： { "Access文件绝对路径": { "Access表名": { SQL目标配置 } } }
# 这样即使不同文件有相同的表名，也能区分开
SYNC_MAPPING = {
# 第一个 Access 文件
r"\\192.168.110.114\生产进度表\2025年数据\生产合同数据.accdb": {
"26年压力表合同数据": {
"target_schema": "productionContractData",
"target_table": "26年压力表合同数据", # 映射到 A 表
"target_table": "26年压力表合同数据",
"pk_col": "ID"
},
"26年温度计合同数据": {
@@ -51,7 +27,6 @@ SYNC_MAPPING = {
"target_table": "25年温度计合同数据",
"pk_col": "ID"
}
},
r"\\192.168.110.114\生产进度表\2026年数据\成品入库.accdb": {
"成品交检记录": {
@@ -323,17 +298,153 @@ SYNC_MAPPING = {
}
}
}
# ================= ntfy 配置 =================
NTFY_CONFIG = {
'enabled': True, # 是否启用通知
'server_url': 'https://ntfy.server10086.icu', # 如果是自建服务器，请修改为自己的 URL
'topic': 'bld', # 你的订阅主题
'token': 'tk_eop5fs66acxtwxf6vlkiojhdvkgb0', # <--- 在这里填入你的 Access Token
'priority': {
'error': 'high', # 错误消息优先级
'critical': 'urgent' # 严重错误优先级
# ================= Excel 文件配置 =================
# 从原 update_config.py 抽取
EXCEL_CONFIGS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2022",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2025",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2026",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
}
# ================= 运行参数 =================
POLL_INTERVAL = 5 # 轮询间隔(秒)
BATCH_SIZE = 10000 # 批量处理大小
]
# ================= 迁移任务配置 =================
# 从原 migration.py 抽取
MIGRATION_TASKS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"year": 2022,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"year": 2025,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"year": 2026,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": "车间号",
"工令号": "工令号",
"客户型号": "客户型号"
}
}
]

9
db_utils.py
View File

@@ -1,21 +1,20 @@
# db_utils.py
import pyodbc
import config
from config import SQL_SERVER_CONN, ACCESS_DRIVER
def get_sql_conn():
"""获取 SQL Server 连接"""
c = config.SQL_SERVER_CONN
# 显式添加 TrustServerCertificate=yes 以兼容 ODBC Driver 18+
conn_str = (
f"DRIVER={c['driver']};SERVER={c['server']};"
f"DATABASE={c['database']};UID={c['uid']};PWD={c['pwd']};"
f"DRIVER={SQL_SERVER_CONN['driver']};SERVER={SQL_SERVER_CONN['server']};"
f"DATABASE={SQL_SERVER_CONN['database']};UID={SQL_SERVER_CONN['uid']};PWD={SQL_SERVER_CONN['pwd']};"
"Encrypt=yes;TrustServerCertificate=yes;"
)
return pyodbc.connect(conn_str)
def get_access_conn(file_path):
"""获取 Access 连接"""
conn_str = f"DRIVER={config.ACCESS_DRIVER};DBQ={file_path};"
conn_str = f"DRIVER={ACCESS_DRIVER};DBQ={file_path};"
return pyodbc.connect(conn_str)
def fmt_table(schema, table):

26
etl_manager.py
View File

@@ -13,7 +13,7 @@ from sqlalchemy.engine import URL
from sqlalchemy.types import NVARCHAR, Integer, Date
# 导入配置
import update_config as config
from config import DB_CONFIG, CACHE_DIR, EXCEL_CONFIGS, BATCH_SIZE, TABLE_SCHEMA
# ================= 抑制 openpyxl 的数据验证警告 =================
warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl')
@@ -37,19 +37,19 @@ class DataSynchronizer:
def __init__(self, force_sync=False):
self.force_sync = force_sync
self.engine = self._get_db_connection()
self.cache_dir = config.CACHE_DIR
self.cache_dir = CACHE_DIR
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
def _get_db_connection(self):
connection_string = (
f"DRIVER={{{config.DB_CONFIG['driver']}}};"
f"SERVER={config.DB_CONFIG['server']};"
f"DATABASE={config.DB_CONFIG['database']};"
f"UID={config.DB_CONFIG['username']};"
f"PWD={config.DB_CONFIG['password']};"
f"TrustServerCertificate={config.DB_CONFIG.get('TrustServerCertificate', 'no')};"
f"DRIVER={{{DB_CONFIG['driver']}}};"
f"SERVER={DB_CONFIG['server']};"
f"DATABASE={DB_CONFIG['database']};"
f"UID={DB_CONFIG['username']};"
f"PWD={DB_CONFIG['password']};"
f"TrustServerCertificate={DB_CONFIG.get('TrustServerCertificate', 'no')};"
)
connection_url = URL.create("mssql+pyodbc", query={"odbc_connect": connection_string})
return create_engine(connection_url, fast_executemany=True)
@@ -94,7 +94,7 @@ class DataSynchronizer:
df = df.drop_duplicates(subset=['总排号'], keep='first')
# 3. 补全列
for col in config.TABLE_SCHEMA.keys():
for col in TABLE_SCHEMA.keys():
if col not in df.columns:
df[col] = None
@@ -102,7 +102,7 @@ class DataSynchronizer:
dtype_dict = {}
# 4. 字段清洗
for col, rules in config.TABLE_SCHEMA.items():
for col, rules in TABLE_SCHEMA.items():
if col not in df.columns:
continue
@@ -136,7 +136,7 @@ class DataSynchronizer:
dtype_dict[col] = NVARCHAR(max_len)
final_cols = list(config.TABLE_SCHEMA.keys())
final_cols = list(TABLE_SCHEMA.keys())
return df[final_cols], dtype_dict
@@ -162,7 +162,7 @@ class DataSynchronizer:
if not df_insert.empty:
logger.info("正在执行批量插入...")
df_insert.to_sql('executionCardData', self.engine, schema='warehouseOutbound',
if_exists='append', index=False, chunksize=config.BATCH_SIZE,
if_exists='append', index=False, chunksize=BATCH_SIZE,
dtype=dtype_dict)
logger.info("批量插入完成。")
@@ -198,7 +198,7 @@ class DataSynchronizer:
logger.info(f"批量更新完成，共影响 {affected_rows} 行。")
def process_excel_files(self):
for cfg in config.EXCEL_CONFIGS:
for cfg in EXCEL_CONFIGS:
remote_path = cfg['file_path']
filename = os.path.basename(remote_path)
local_path = os.path.join(self.cache_dir, filename)

10
init_full_sync.py
View File

@@ -1,5 +1,5 @@
import pyodbc
import config
from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, BATCH_SIZE
import db_utils
import time
import os
@@ -103,8 +103,8 @@ def run_full_sync():
logger.info("=" * 70)
log_start(logger, "全量初始化同步")
log_info(logger, f"配置文件数: {len(config.SYNC_MAPPING)} 个")
log_info(logger, f"批次大小: {config.BATCH_SIZE} 行")
log_info(logger, f"配置文件数: {len(SYNC_MAPPING)} 个")
log_info(logger, f"批次大小: {BATCH_SIZE} 行")
logger.info("=" * 70)
sync_start_time = time.time()
@@ -123,7 +123,7 @@ def run_full_sync():
return
# 第一层循环:遍历配置文件中的所有文件路径
for acc_path, tables_map in config.SYNC_MAPPING.items():
for acc_path, tables_map in SYNC_MAPPING.items():
if not os.path.exists(acc_path):
log_skip(logger, f"文件不存在: {acc_path}")
continue
@@ -184,7 +184,7 @@ def run_full_sync():
last_log_time = start_time
while True:
rows = acc_cursor.fetchmany(config.BATCH_SIZE)
rows = acc_cursor.fetchmany(BATCH_SIZE)
if not rows: break
sql_cursor.executemany(insert_sql, rows)

85
migration.py
View File

@@ -3,21 +3,13 @@ import os
import shutil
import urllib
from sqlalchemy import create_engine, text
from config import DB_CONFIG, MIGRATION_TASKS, TEMP_DIR
import ntfy_utils # 确保该文件在同一目录下
# ==========================================
# 1. 脚本配置 (Configuration)
# ==========================================
# 数据库连接信息
DB_CONFIG = {
"server": "192.168.110.114",
"database": "CompanyDB",
"username": "peng",
"password": "Cqbld123456.",
"driver": "ODBC Driver 18 for SQL Server"
}
# 目标表配置
TARGET_DB_SCHEMA = "warehouseOutbound"
TARGET_TABLE_NAME = "customerProductType"
@@ -31,81 +23,6 @@ SQL_COL_MODEL = "客户型号"
# 运行参数
FORCE_UPDATE = False # 如果设为 True，则无视时间对比，强制更新所有文件
TEMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "temp")
# 迁移任务清单
MIGRATION_TASKS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"year": 2022,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": SQL_COL_WORKSHOP,
"工令号": SQL_COL_ORDER,
"客户型号": SQL_COL_MODEL
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": SQL_COL_WORKSHOP,
"工令号": SQL_COL_ORDER,
"客户型号": SQL_COL_MODEL
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"year": 2023,
"sheet_names": ["Sheet1"],
"mapping": {
"车间号": SQL_COL_WORKSHOP,
"工令号": SQL_COL_ORDER,
"客户型号": SQL_COL_MODEL
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": SQL_COL_WORKSHOP,
"工令号": SQL_COL_ORDER,
"客户型号": SQL_COL_MODEL
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"year": 2024,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": SQL_COL_WORKSHOP,
"工令号": SQL_COL_ORDER,
"客户型号": SQL_COL_MODEL
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"year": 2025,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": SQL_COL_WORKSHOP,
"工令号": SQL_COL_ORDER,
"客户型号": SQL_COL_MODEL
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"year": 2026,
"sheet_names": ["重庆数据","北京数据"],
"mapping": {
"车间号": SQL_COL_WORKSHOP,
"工令号": SQL_COL_ORDER,
"客户型号": SQL_COL_MODEL
}
}
]
# ==========================================
# 2. 核心辅助函数

8
ntfy_utils.py
View File

@@ -1,12 +1,12 @@
# ntfy_utils.py
import requests
import config
from config import NTFY_CONFIG
def send_ntfy(message, title="数据库同步消息", priority="default", tags=None):
"""
向加密的 ntfy 服务器发送消息
"""
conf = config.NTFY_CONFIG
conf = NTFY_CONFIG
if not conf.get('enabled', False):
return
@@ -51,7 +51,7 @@ def send_error(msg):
send_ntfy(
message=str(msg),
title="❌ 同步任务错误",
priority=config.NTFY_CONFIG['priority']['error'],
priority=NTFY_CONFIG['priority']['error'],
tags=["warning", "database"]
)
@@ -60,6 +60,6 @@ def send_critical(msg):
send_ntfy(
message=str(msg),
title="🔥 同步服务崩溃",
priority=config.NTFY_CONFIG['priority']['critical'],
priority=NTFY_CONFIG['priority']['critical'],
tags=["skull", "critical"]
)

14
run_incremental_sync.py
View File

@@ -2,7 +2,7 @@ import time
import os
import sys
import pyodbc
import config
from config import SQL_SERVER_CONN, ACCESS_DRIVER, SYNC_MAPPING, POLL_INTERVAL, LOG_TABLE_CONFIG
import db_utils
import logging
from logging.handlers import TimedRotatingFileHandler
@@ -95,7 +95,7 @@ def log_sync(message):
def process_sync_task():
"""
逻辑重构：
遍历 config.SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。
遍历 SYNC_MAPPING 中的每一个文件 -> 去日志表查询该文件下特定表的未同步记录。
"""
sql_conn = db_utils.get_sql_conn()
sql_cursor = sql_conn.cursor()
@@ -104,12 +104,12 @@ def process_sync_task():
# 标记是否有工作被处理（用于控制轮询休眠时间）
work_done = False
cols = config.LOG_TABLE_CONFIG
cols = LOG_TABLE_CONFIG
log_full_name = db_utils.fmt_table(cols['schema'], cols['table_name'])
try:
# === 核心循环：以配置文件为驱动 ===
for clean_path, tables_map in config.SYNC_MAPPING.items():
for clean_path, tables_map in SYNC_MAPPING.items():
# 1. 准备查询条件
# 获取该文件下所有需要同步的表名列表
@@ -265,8 +265,8 @@ def process_sync_task():
if __name__ == "__main__":
log_start("增量同步服务已启动 (配置驱动模式)")
log_info(f"轮询间隔: {config.POLL_INTERVAL} 秒")
log_info(f"监控配置: {len(config.SYNC_MAPPING)} 个文件")
log_info(f"轮询间隔: {POLL_INTERVAL} 秒")
log_info(f"监控配置: {len(SYNC_MAPPING)} 个文件")
logger.info("=" * 70)
while True:
@@ -274,7 +274,7 @@ if __name__ == "__main__":
has_work = process_sync_task()
# 如果有工作，说明可能还有积压，休息短一点(0.1s)
# 如果没工作，休息标准间隔(5s)
time.sleep(0.1 if has_work else config.POLL_INTERVAL)
time.sleep(0.1 if has_work else POLL_INTERVAL)
except KeyboardInterrupt:
logger.info("=" * 70)
log_stop("收到停止信号，服务正在关闭...")

154
update_config.py
View File

@@ -1,154 +0,0 @@
import os
# ================= 数据库配置 =================
DB_CONFIG = {
"server": "192.168.110.114",
"database": "CompanyDB",
"username": "peng",
"password": "Cqbld123456.",
"driver": "ODBC Driver 18 for SQL Server",
"TrustServerCertificate": "yes"
}
# ================= 全局配置 =================
CACHE_DIR = os.path.join(os.getcwd(), "temp")
BATCH_SIZE = 5000
# ================= 字段清洗规则 =================
# 基于 NVARCHAR (按字符数计算长度)
TABLE_SCHEMA = {
"合同年份": {"type": "str", "max_len": 10},
"总排号": {"type": "str", "max_len": 50},
"序号": {"type": "int"},
"订单号": {"type": "str", "max_len": 150},
"车间号": {"type": "str", "max_len": 20},
"销售内部号": {"type": "str", "max_len": 50},
"经办人": {"type": "str", "max_len": 20},
"签订日期": {"type": "date"},
"交货日期": {"type": "date"},
"客户名称": {"type": "str", "max_len": 200},
"产品名称": {"type": "str", "max_len": 200},
"客户型号": {"type": "str", "max_len": 200},
"选型型号": {"type": "str", "max_len": 200},
"量程": {"type": "str", "max_len": 150},
"数量": {"type": "int"},
"备注2": {"type": "str", "max_len": 500},
"备注1": {"type": "str", "max_len": 500},
"位号": {"type": "str", "max_len": 500},
"技术参数": {"type": "str", "max_len": 1000},
"车间": {"type": "str", "max_len": 200},
"工令号": {"type": "str", "max_len": 200},
"接单日期": {"type": "date"},
"新参数": {"type": "str", "max_len": 1000},
"基本型号": {"type": "str", "max_len": 200},
"公称外径": {"type": "str", "max_len": 200},
"安装代码": {"type": "str", "max_len": 200},
"设计形式": {"type": "str", "max_len": 200},
"技术安装代码": {"type": "str", "max_len": 200},
"隔膜类型": {"type": "str", "max_len": 200},
"标准": {"type": "str", "max_len": 100},
"隔膜大小": {"type": "str", "max_len": 200},
"隔膜材质": {"type": "str", "max_len": 200},
"膜片": {"type": "str", "max_len": 200},
"膜片材质": {"type": "str", "max_len": 200},
"CRM明细ID号": {"type": "str", "max_len": 200},
"成品物料编码": {"type": "str", "max_len": 200},
"工单号": {"type": "str", "max_len": 200},
"盘号": {"type": "str", "max_len": 200},
"编码": {"type": "str", "max_len": 200},
"型批名称": {"type": "str", "max_len": 200},
"型批型号": {"type": "str", "max_len": 200},
"开票名称": {"type": "str", "max_len": 200},
"开票型号": {"type": "str", "max_len": 200},
"上数据时间": {"type": "date"},
"生产代码1": {"type": "str", "max_len": 50},
"生产代码2": {"type": "str", "max_len": 50}
}
CONTRACT_MAPPING = {
"合同年份": "合同年份",
"车间号": "车间号",
"工令号": "工令号",
"订单号": "订单号",
"客户名称": "客户名称",
"产品型号": "选型型号",
"量程": "量程",
"数量": "数量",
"单价": None,
"ID": None,
"位号": "位号"
}
EXCEL_CONFIGS = [
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2022.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2022",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023（1-5月）.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2023(6月-.xlsm",
"sheet_names": ["Sheet1"],
"contract_year": "2023",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024 6月.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\往年生产执行卡\生产执行卡2024.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2024",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2025年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2025",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
},
{
"file_path": r"\\192.168.110.113\生产执行卡\生产执行卡2026年.xlsm",
"sheet_names": ["重庆数据","北京数据"],
"contract_year": "2026",
"field_mapping": {
"产品型号": "选型型号",
"备注": "备注1",
"下单日期": "接单日期"
}
}
]