feat: add SQL Server database persistence for extracted material plan data

Add optional database persistence feature that automatically saves extracted
discrete material plan data to SQL Server. Users can enable this feature in
the settings tab.

Changes:
- Add enable_db_persistence flag to ExtractionConfig (default: disabled)
- Create DiscreteMaterialPlanDAO for database operations with REPLACE pattern
- Update progress tracking to include database persistence stage (90-100%)
- Add database persistence checkbox in settings UI
- Remove verbose logging checkbox from data extraction UI (config-only now)
- Update extraction workflow to save merged DataFrame to database

Progress weights adjusted:
- download: 65% -> 60%
- database: 10% (new stage)
- Other stages adjusted accordingly

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-02-06 12:18:15 +08:00
parent 3180ccacb8
commit 53a1e33e45
8 changed files with 394 additions and 24 deletions

View File

@@ -19,7 +19,8 @@ class DiscreteMaterialPlanExtractor:
"""离散备料计划维护数据提取器"""
def __init__(
self, username, password, headless=False, verbose=True, batch_size=100
self, username, password, headless=False, verbose=True, batch_size=100,
enable_db_persistence=False
):
"""
初始化提取器
@@ -30,6 +31,7 @@ class DiscreteMaterialPlanExtractor:
headless: 是否无头模式运行
verbose: 是否打印详细日志
batch_size: 批次大小
enable_db_persistence: 是否启用数据库持久化
"""
self.username = username
self.password = password
@@ -38,6 +40,12 @@ class DiscreteMaterialPlanExtractor:
self.batch_size = batch_size
self.progress_callback = None
self.converter = ExcelConverter(verbose=verbose)
self.enable_db_persistence = enable_db_persistence
self.dao = None
if self.enable_db_persistence:
from db.discrete_material_plan_dao import DiscreteMaterialPlanDAO
self.dao = DiscreteMaterialPlanDAO()
self.dao.__enter__() # Enter context manager
def _print(self, *args, **kwargs):
"""打印日志(如果 verbose=True"""
@@ -269,7 +277,7 @@ class DiscreteMaterialPlanExtractor:
return download_path
def convert_and_merge_files(self, file_paths, output_path):
"""使用 ExcelConverter 转换并合并所有文件"""
"""使用 ExcelConverter 转换并合并所有文件,返回合并后的 DataFrame"""
# 确保输出文件路径是正确的格式
output_path = os.path.normpath(output_path)
output_dir = os.path.dirname(output_path)
@@ -287,7 +295,7 @@ class DiscreteMaterialPlanExtractor:
"准备转换:检查输出目录",
action="check_directory",
)
if output_dir and not os.path.exists(output_dir):
self._print(f"创建输出目录: {output_dir}")
os.makedirs(output_dir)
@@ -312,7 +320,7 @@ class DiscreteMaterialPlanExtractor:
df = self.converter.convert(file_path, output_file=None) # 只转换,不保存
all_dataframes.append(df)
self._print(f" 提取到 {len(df)} 条记录")
# 报告转换完成
self._report_progress(
"convert",
@@ -324,6 +332,7 @@ class DiscreteMaterialPlanExtractor:
action="file_converted",
)
merged_df = None
if all_dataframes:
# 步骤N+1合并数据
self._report_progress(
@@ -334,7 +343,7 @@ class DiscreteMaterialPlanExtractor:
action="merging_data",
file_count=len(all_dataframes),
)
self._print(f"\n合并 {len(all_dataframes)} 个文件的数据...")
merged_df = pd.concat(all_dataframes, ignore_index=True)
merged_df.to_excel(output_path, index=False)
@@ -349,13 +358,43 @@ class DiscreteMaterialPlanExtractor:
action="cleanup",
total_records=len(merged_df),
)
for file_path in file_paths:
os.remove(file_path)
self._print(f"已删除临时文件: {file_path}")
return output_path
return None
return output_path, merged_df
return None, None
def _save_to_database(self, df: pd.DataFrame):
"""Save DataFrame to database with progress reporting"""
try:
self._report_progress(
"database", 0, 3, "准备保存到数据库...",
action="db_start"
)
stats = self.dao.save_dataframe_with_replace(df)
self._report_progress(
"database", 3, 3,
f"数据库保存完成: 删除 {stats['deleted']} 条, 新增 {stats['inserted']}",
action="db_complete",
stats=stats
)
self._print(f"\n数据库保存成功:")
self._print(f" 删除旧记录: {stats['deleted']}")
self._print(f" 新增记录: {stats['inserted']}")
except Exception as e:
self._print(f"\n警告: 数据库保存失败: {e}")
self._report_progress(
"database", 3, 3,
f"数据库保存失败: {str(e)}",
action="db_error",
error=str(e)
)
def setup_query_interface(self, inner_frame):
"""设置查询界面(不报告进度,由 extract 统一报告)"""
@@ -506,7 +545,12 @@ class DiscreteMaterialPlanExtractor:
self._print(
f"\n=== 开始转换并合并 {len(downloaded_files)} 个文件 ==="
)
self.convert_and_merge_files(downloaded_files, output_file)
output_path, merged_df = self.convert_and_merge_files(downloaded_files, output_file)
# 数据库保存步骤(独立阶段)
if self.enable_db_persistence and self.dao and merged_df is not None:
self._print(f"\n=== 开始保存数据到数据库 ===")
self._save_to_database(merged_df)
else:
self._print("\n没有下载到任何文件")
@@ -514,7 +558,7 @@ class DiscreteMaterialPlanExtractor:
self._print(f"最终文件: {output_file}")
self._report_progress(
"complete", 1, 1, "数据提取完成 ✓",
"complete", 1, 1, "数据提取完成 ✓",
output_file=output_file,
action="all_complete",
)
@@ -525,6 +569,12 @@ class DiscreteMaterialPlanExtractor:
return output_file
finally:
# Close database connection if open
if self.dao:
try:
self.dao.__exit__(None, None, None)
except Exception:
pass
self.progress_callback = original_callback