admin/playwrite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add progress callback mechanism for real-time updates in data extraction

Browse Source 
- Add ProgressInfo and ProgressCalculator classes for structured progress tracking
- Implement RealtimeOutput for immediate log display without buffering
- Add progress callback support to DiscreteMaterialPlanExtractor
- Use queue-based thread-safe communication for GUI progress updates
- Fix log output issue where all content appeared at once after completion

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-02-05 14:06:30 +08:00
parent 71621dc8a0
commit 885d87dff8
4 changed files with 329 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
gui/data_extraction_tab.py
View File

@@ -9,13 +9,15 @@
import os
import sys
import threading
import queue
import tkinter as tk
from tkinter import ttk, filedialog, messagebox
from pathlib import Path
from io import StringIO
from contextlib import redirect_stdout
from gui.widgets import FileSelector, LogText
from gui.config_manager import ConfigManager
from gui.progress import ProgressInfo, ProgressCalculator
from gui.utils import RealtimeOutput
class DataExtractionTab(ttk.Frame):
@@ -34,6 +36,11 @@ class DataExtractionTab(ttk.Frame):
self.extracting = False
self.extractor = None
self.extraction_thread = None
self.progress_calculator = ProgressCalculator()
self.progress_queue = queue.Queue() # 进度更新队列
# 启动进度更新轮询
self._poll_progress_queue()
self.create_widgets()
@@ -163,6 +170,7 @@ class DataExtractionTab(ttk.Frame):
self.start_button.config(state=tk.DISABLED)
self.stop_button.config(state=tk.NORMAL)
self.progress_bar['value'] = 0
self.status_label.config(text="正在登录...")
self.log_text.clear()
self.log_text.info("开始数据提取...")
@@ -195,25 +203,24 @@ class DataExtractionTab(ttk.Frame):
verbose=self.verbose_var.get()
)
# 捕获 stdout 输出
captured_output = StringIO()
# 创建实时输出流，每次写入立即更新 GUI
realtime_output = RealtimeOutput(lambda line: self._update_log(line, "INFO"))
# 重定向 stdout 并执行提取
with redirect_stdout(captured_output):
# 创建进度回调函数
def progress_callback(progress_info: ProgressInfo):
# 计算总体进度百分比
overall_percent = self.progress_calculator.calculate_overall_percent(progress_info)
self._update_progress(overall_percent, progress_info.message)
# 重定向 stdout 并执行提取（带进度回调）
with redirect_stdout(realtime_output):
result = self.extractor.extract(
production_id_file=input_file,
output_file=output_file
output_file=output_file,
progress_callback=progress_callback
)
# 获取捕获的输出并显示到日志
output_text = captured_output.getvalue()
if output_text:
for line in output_text.split('\n'):
if line.strip():
self._update_log(line, "INFO")
if result and self.extracting:
self._update_progress(100, "提取完成")
self._update_log(f"数据已保存到：{output_file}", "SUCCESS")
elif not self.extracting:
self._update_log("提取已取消", "WARNING")
@@ -233,14 +240,28 @@ class DataExtractionTab(ttk.Frame):
self.stop_button.config(state=tk.DISABLED)
self.extractor = None
def _update_progress(self, value: int, message: str):
"""线程安全的进度更新"""
def update():
if self.extracting:
self.progress_bar['value'] = value
self.status_label.config(text=message)
def _poll_progress_queue(self):
"""轮询进度队列，处理进度更新"""
try:
while True:
# 非阻塞地获取队列中的消息
try:
progress_data = self.progress_queue.get_nowait()
value, message = progress_data
self.progress_bar['value'] = value
self.status_label.config(text=message)
except queue.Empty:
break
finally:
# 继续轮询（每 50ms 检查一次）
self.after(50, self._poll_progress_queue)
self.after(0, update)
def _update_progress(self, value: int, message: str):
"""线程安全的进度更新（通过队列）"""
try:
self.progress_queue.put_nowait((value, message))
except:
pass # 队列满时忽略
def _update_log(self, message: str, level: str = "INFO"):
"""线程安全的日志更新"""

97
gui/progress.py Normal file
View File

@@ -0,0 +1,97 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
进度信息模块
定义用于进度回调的数据结构。
"""
from dataclasses import dataclass, field
from typing import Dict, Any, Optional
@dataclass
class ProgressInfo:
"""
进度信息
用于在后台任务和 GUI 之间传递进度信息。
"""
stage: str # 阶段标识: 'login', 'query', 'download', 'logout', 'convert', 'complete'
current: int # 当前进度值
total: int # 总量
message: str # 显示给用户的消息
detail: Dict[str, Any] = field(default_factory=dict) # 额外详细信息
@property
def percent(self) -> int:
"""计算进度百分比（0-100）"""
if self.total <= 0:
return 0
return int(self.current * 100 / self.total)
def __repr__(self) -> str:
return f"ProgressInfo(stage={self.stage}, {self.current}/{self.total}, {self.message})"
class ProgressCalculator:
"""
进度计算器
将各阶段的进度映射到总体进度百分比。
"""
# 各阶段在总进度中的占比
STAGE_WEIGHTS = {
'login': 5, # 登录: 0-5%
'query': 5, # 查询: 5-10%
'download': 65, # 下载: 10-75%
'logout': 5, # 注销: 75-80%
'convert': 15, # 转换: 80-95%
'complete': 5, # 完成: 95-100%
}
def __init__(self):
"""初始化进度计算器"""
self._stage_offsets = self._calculate_stage_offsets()
def _calculate_stage_offsets(self) -> Dict[str, int]:
"""计算各阶段的起始偏移量（百分比）"""
offsets = {}
offset = 0
for stage, weight in self.STAGE_WEIGHTS.items():
offsets[stage] = offset
offset += weight
return offsets
def calculate_overall_percent(self, progress: ProgressInfo) -> int:
"""
计算总体进度百分比
Args:
progress: 进度信息
Returns:
总体进度百分比 (0-100)
"""
stage = progress.stage
if stage == 'complete':
return 100
if stage not in self._stage_offsets:
return 0
# 计算阶段起始百分比
stage_offset = self._stage_offsets[stage]
# 计算阶段内的进度百分比
stage_percent = progress.percent
# 计算该阶段的权重
stage_weight = self.STAGE_WEIGHTS[stage]
# 总进度 = 阶段偏移 + (阶段内进度 * 阶段权重 / 100)
overall = stage_offset + int(stage_percent * stage_weight / 100)
return min(overall, 100)

39
gui/utils.py Normal file
View File

@@ -0,0 +1,39 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
GUI 工具模块
提供 GUI 相关的工具类和函数。
"""
class RealtimeOutput:
"""实时输出流，每次写入立即回调通知"""
def __init__(self, callback):
"""
初始化实时输出流
Args:
callback: 写入时的回调函数，接收文本内容
"""
self.callback = callback
self.buffer = []
def write(self, text):
"""写入文本"""
if text:
# 将文本按行分割，逐行回调
lines = text.split('\n')
for line in lines:
if line: # 忽略空行（由 split 产生）
self.callback(line)
return len(text)
def flush(self):
"""刷新（兼容性方法）"""
pass
def isatty(self):
"""返回 False，表示不是终端"""
return False

212
utils/离散备料计划维护数据提取.py
View File

@@ -8,12 +8,13 @@ from playwright.sync_api import sync_playwright
from utils.excel_converter import ExcelConverter
from utils.auth import login, logout
from db.production_order_query import read_production_ids, query_production_order_numbers
from typing import Callable, Optional
class DiscreteMaterialPlanExtractor:
"""离散备料计划维护数据提取器"""
def __init__(self, username, password, headless=False, verbose=True):
def __init__(self, username, password, headless=False, verbose=True, progress_callback=None):
"""
初始化提取器
@@ -22,11 +23,13 @@ class DiscreteMaterialPlanExtractor:
password: 登录密码
headless: 是否无头模式运行
verbose: 是否打印详细日志
progress_callback: 进度回调函数，接收 ProgressInfo 对象
"""
self.username = username
self.password = password
self.headless = headless
self.verbose = verbose
self.progress_callback = progress_callback
self.converter = ExcelConverter(verbose=verbose)
def _print(self, *args, **kwargs):
@@ -34,12 +37,39 @@ class DiscreteMaterialPlanExtractor:
if self.verbose:
print(*args, **kwargs)
def get_production_order_numbers(self, production_id_file):
def _report_progress(self, stage: str, current: int, total: int, message: str, **detail):
"""
报告进度
Args:
stage: 阶段标识
current: 当前进度值
total: 总量
message: 显示消息
**detail: 额外详细信息
"""
if self.progress_callback:
try:
from gui.progress import ProgressInfo
progress_info = ProgressInfo(
stage=stage,
current=current,
total=total,
message=message,
detail=detail
)
self.progress_callback(progress_info)
except Exception:
# 如果进度回调失败，忽略错误，不影响主流程
pass
def get_production_order_numbers(self, production_id_file, report_progress=False):
"""
读取总排号文件并查询数据库获取生产订单号
Args:
production_id_file: ProductionID.txt 文件路径
report_progress: 是否报告进度
Returns:
生产订单号列表
@@ -52,6 +82,9 @@ class DiscreteMaterialPlanExtractor:
order_ids = query_production_order_numbers(production_ids)
self._print(f"查询到 {len(order_ids)} 个生产订单号")
if report_progress:
self._report_progress('query', 1, 1, f"查询到 {len(order_ids)} 个生产订单号", count=len(order_ids))
return order_ids
def group_order_ids(self, order_ids, group_size=100):
@@ -59,7 +92,7 @@ class DiscreteMaterialPlanExtractor:
for i in range(0, len(order_ids), group_size):
yield order_ids[i:i + group_size]
def download_batch(self, inner_frame, order_ids, batch_index, page1, debug_mode=False, debug_batch=None):
def download_batch(self, inner_frame, order_ids, batch_index, total_batches, page1, debug_mode=False, debug_batch=None):
"""下载一批订单号的数据"""
from playwright.sync_api import TimeoutError
import re
@@ -111,6 +144,15 @@ class DiscreteMaterialPlanExtractor:
download.save_as(download_path)
self._print(f"第 {batch_index + 1} 批下载完成: {download_path}")
# 报告进度
self._report_progress(
'download',
batch_index + 1,
total_batches,
f"第 {batch_index + 1}/{total_batches} 批下载完成",
batch_index=batch_index + 1
)
# 关闭输出对话框（如果有的话）
# try:
# inner_frame.get_by_role("button", name="取消").click()
@@ -142,6 +184,17 @@ class DiscreteMaterialPlanExtractor:
for i, file_path in enumerate(file_paths, 1):
self._print(f"转换第 {i} 个文件: {file_path}")
# 报告进度
self._report_progress(
'convert',
i,
len(file_paths),
f"转换第 {i}/{len(file_paths)} 个文件",
file_index=i,
file_path=file_path
)
df = self.converter.convert(file_path, output_file=None) # 只转换，不保存
all_dataframes.append(df)
self._print(f" 提取到 {len(df)} 条记录")
@@ -187,7 +240,7 @@ class DiscreteMaterialPlanExtractor:
def extract(self, production_id_file, data_dir="D:/python/playwrite/data",
output_file="D:/python/playwrite/data/离散备料计划维护_合并.xlsx",
debug_mode=False, debug_batch=None):
debug_mode=False, debug_batch=None, progress_callback=None):
"""
执行完整的数据提取流程
@@ -197,76 +250,113 @@ class DiscreteMaterialPlanExtractor:
output_file: 最终输出文件路径
debug_mode: 是否启用调试模式
debug_batch: 调试批次号
progress_callback: 进度回调函数（覆盖初始化时的回调）
Returns:
输出文件路径
"""
with sync_playwright() as playwright:
# 调用登录模块
browser, context, page, main_frame = login(
playwright=playwright,
username=self.username,
password=self.password,
headless=self.headless,
ignore_https_errors=True
)
# 保存原始回调
original_callback = self.progress_callback
# 使用传入的回调或初始化时的回调
self.progress_callback = progress_callback or self.progress_callback
self._print("=" * 80)
self._print("开始执行离散备料计划维护数据提取")
self._print("=" * 80)
# 登录成功后可以进行后续操作
# 点击打开"功能菜单"
main_frame.locator("i").first.click()
# 点击打开"离散备料计划维护"
with page.expect_popup() as page1_info:
main_frame.get_by_title("离散备料计划维护", exact=True).first.click()
page1 = page1_info.value
# 获取 nested iframe
main_frame = page1.locator("#forwardFrame").content_frame
inner_frame_locator = main_frame.locator("#mainiframe")
inner_frame_locator.wait_for(state="visible", timeout=15000)
inner_frame = inner_frame_locator.content_frame
# 设置查询界面
self.setup_query_interface(inner_frame)
# 读取总排号并查询生产订单号
order_ids = self.get_production_order_numbers(production_id_file)
# 按批次下载
downloaded_files = []
for batch_index, order_ids_batch in enumerate(self.group_order_ids(order_ids, 100)):
self._print(f"\n=== 开始处理第 {batch_index + 1} 批，共 {len(order_ids_batch)} 个订单号 ===")
downloaded_file = self.download_batch(
inner_frame, order_ids_batch, batch_index, page1,
debug_mode=debug_mode, debug_batch=debug_batch
try:
with sync_playwright() as playwright:
# 调用登录模块
browser, context, page, main_frame = login(
playwright=playwright,
username=self.username,
password=self.password,
headless=self.headless,
ignore_https_errors=True
)
downloaded_files.append(downloaded_file)
# 执行账号注销
self._print("\n开始执行账号注销...")
logout(main_frame, verbose=self.verbose)
# 登录完成
self._report_progress('login', 1, 1, "登录成功")
# 转换并合并文件
if downloaded_files:
self._print(f"\n=== 开始转换并合并 {len(downloaded_files)} 个文件 ===")
self.convert_and_merge_files(downloaded_files, output_file)
else:
self._print("\n没有下载到任何文件")
self._print("=" * 80)
self._print("开始执行离散备料计划维护数据提取")
self._print("=" * 80)
self._print(f"\n=== 全部完成 ===")
self._print(f"最终文件: {output_file}")
# 登录成功后可以进行后续操作
# 点击打开"功能菜单"
main_frame.locator("i").first.click()
# 点击打开"离散备料计划维护"
with page.expect_popup() as page1_info:
main_frame.get_by_title("离散备料计划维护", exact=True).first.click()
page1 = page1_info.value
# 获取 nested iframe
main_frame = page1.locator("#forwardFrame").content_frame
inner_frame_locator = main_frame.locator("#mainiframe")
inner_frame_locator.wait_for(state="visible", timeout=15000)
inner_frame = inner_frame_locator.content_frame
# 关闭浏览器
context.close()
browser.close()
# 设置查询界面
self.setup_query_interface(inner_frame)
return output_file
# 读取总排号并查询生产订单号
order_ids = self.get_production_order_numbers(production_id_file, report_progress=True)
# 按批次下载
downloaded_files = []
# 计算总批次数
total_batches = sum(1 for _ in self.group_order_ids(order_ids, 100))
for batch_index, order_ids_batch in enumerate(self.group_order_ids(order_ids, 100)):
self._print(f"\n=== 开始处理第 {batch_index + 1} 批，共 {len(order_ids_batch)} 个订单号 ===")
# 报告开始下载批次
self._report_progress(
'download',
batch_index,
total_batches,
f"正在下载第 {batch_index + 1}/{total_batches} 批...",
batch_index=batch_index + 1,
batch_size=len(order_ids_batch)
)
downloaded_file = self.download_batch(
inner_frame, order_ids_batch, batch_index, total_batches, page1,
debug_mode=debug_mode, debug_batch=debug_batch
)
downloaded_files.append(downloaded_file)
# 执行账号注销
self._print("\n开始执行账号注销...")
self._report_progress('logout', 1, 1, "正在注销账号...")
logout(main_frame, verbose=self.verbose)
# 转换并合并文件
if downloaded_files:
self._report_progress(
'convert',
0,
len(downloaded_files),
f"开始转换并合并 {len(downloaded_files)} 个文件",
file_count=len(downloaded_files)
)
self._print(f"\n=== 开始转换并合并 {len(downloaded_files)} 个文件 ===")
self.convert_and_merge_files(downloaded_files, output_file)
else:
self._print("\n没有下载到任何文件")
self._print(f"\n=== 全部完成 ===")
self._print(f"最终文件: {output_file}")
# 报告完成
self._report_progress('complete', 1, 1, "提取完成", output_file=output_file)
# 关闭浏览器
context.close()
browser.close()
return output_file
finally:
# 恢复原始回调
self.progress_callback = original_callback
def main():
"""测试函数"""