feat: add progress callback mechanism for real-time updates in data extraction

- Add ProgressInfo and ProgressCalculator classes for structured progress tracking
- Implement RealtimeOutput for immediate log display without buffering
- Add progress callback support to DiscreteMaterialPlanExtractor
- Use queue-based thread-safe communication for GUI progress updates
- Fix log output issue where all content appeared at once after completion

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

gui/data_extraction_tab.py

@@ -9,13 +9,15 @@
 import os
 import sys
 import threading
+import queue
 import tkinter as tk
 from tkinter import ttk, filedialog, messagebox
 from pathlib import Path
-from io import StringIO
 from contextlib import redirect_stdout
 from gui.widgets import FileSelector, LogText
 from gui.config_manager import ConfigManager
+from gui.progress import ProgressInfo, ProgressCalculator
+from gui.utils import RealtimeOutput
 
 
 class DataExtractionTab(ttk.Frame):
@@ -34,6 +36,11 @@ class DataExtractionTab(ttk.Frame):
         self.extracting = False
         self.extractor = None
         self.extraction_thread = None
+        self.progress_calculator = ProgressCalculator()
+        self.progress_queue = queue.Queue()  # 进度更新队列
+
+        # 启动进度更新轮询
+        self._poll_progress_queue()
 
         self.create_widgets()
 
@@ -163,6 +170,7 @@ class DataExtractionTab(ttk.Frame):
         self.start_button.config(state=tk.DISABLED)
         self.stop_button.config(state=tk.NORMAL)
         self.progress_bar['value'] = 0
+        self.status_label.config(text="正在登录...")
         self.log_text.clear()
         self.log_text.info("开始数据提取...")
 
@@ -195,25 +203,24 @@ class DataExtractionTab(ttk.Frame):
                 verbose=self.verbose_var.get()
             )
 
-            # 捕获 stdout 输出
+            # 创建实时输出流，每次写入立即更新 GUI
-            captured_output = StringIO()
+            realtime_output = RealtimeOutput(lambda line: self._update_log(line, "INFO"))
 
-            # 重定向 stdout 并执行提取
+            # 创建进度回调函数
-            with redirect_stdout(captured_output):
+            def progress_callback(progress_info: ProgressInfo):
+                # 计算总体进度百分比
+                overall_percent = self.progress_calculator.calculate_overall_percent(progress_info)
+                self._update_progress(overall_percent, progress_info.message)
+
+            # 重定向 stdout 并执行提取（带进度回调）
+            with redirect_stdout(realtime_output):
                 result = self.extractor.extract(
                     production_id_file=input_file,
-                    output_file=output_file
+                    output_file=output_file,
+                    progress_callback=progress_callback
                 )
 
-            # 获取捕获的输出并显示到日志
-            output_text = captured_output.getvalue()
-            if output_text:
-                for line in output_text.split('\n'):
-                    if line.strip():
-                        self._update_log(line, "INFO")
-
             if result and self.extracting:
-                self._update_progress(100, "提取完成")
                 self._update_log(f"数据已保存到：{output_file}", "SUCCESS")
             elif not self.extracting:
                 self._update_log("提取已取消", "WARNING")
@@ -233,14 +240,28 @@ class DataExtractionTab(ttk.Frame):
         self.stop_button.config(state=tk.DISABLED)
         self.extractor = None
 
-    def _update_progress(self, value: int, message: str):
+    def _poll_progress_queue(self):
-        """线程安全的进度更新"""
+        """轮询进度队列，处理进度更新"""
-        def update():
+        try:
-            if self.extracting:
+            while True:
-                self.progress_bar['value'] = value
+                # 非阻塞地获取队列中的消息
-                self.status_label.config(text=message)
+                try:
+                    progress_data = self.progress_queue.get_nowait()
+                    value, message = progress_data
+                    self.progress_bar['value'] = value
+                    self.status_label.config(text=message)
+                except queue.Empty:
+                    break
+        finally:
+            # 继续轮询（每 50ms 检查一次）
+            self.after(50, self._poll_progress_queue)
 
-        self.after(0, update)
+    def _update_progress(self, value: int, message: str):
+        """线程安全的进度更新（通过队列）"""
+        try:
+            self.progress_queue.put_nowait((value, message))
+        except:
+            pass  # 队列满时忽略
 
     def _update_log(self, message: str, level: str = "INFO"):
         """线程安全的日志更新"""

gui/progress.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+进度信息模块
+
+定义用于进度回调的数据结构。
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict, Any, Optional
+
+
+@dataclass
+class ProgressInfo:
+    """
+    进度信息
+
+    用于在后台任务和 GUI 之间传递进度信息。
+    """
+    stage: str  # 阶段标识: 'login', 'query', 'download', 'logout', 'convert', 'complete'
+    current: int  # 当前进度值
+    total: int  # 总量
+    message: str  # 显示给用户的消息
+    detail: Dict[str, Any] = field(default_factory=dict)  # 额外详细信息
+
+    @property
+    def percent(self) -> int:
+        """计算进度百分比（0-100）"""
+        if self.total <= 0:
+            return 0
+        return int(self.current * 100 / self.total)
+
+    def __repr__(self) -> str:
+        return f"ProgressInfo(stage={self.stage}, {self.current}/{self.total}, {self.message})"
+
+
+class ProgressCalculator:
+    """
+    进度计算器
+
+    将各阶段的进度映射到总体进度百分比。
+    """
+
+    # 各阶段在总进度中的占比
+    STAGE_WEIGHTS = {
+        'login': 5,      # 登录: 0-5%
+        'query': 5,      # 查询: 5-10%
+        'download': 65,  # 下载: 10-75%
+        'logout': 5,     # 注销: 75-80%
+        'convert': 15,   # 转换: 80-95%
+        'complete': 5,   # 完成: 95-100%
+    }
+
+    def __init__(self):
+        """初始化进度计算器"""
+        self._stage_offsets = self._calculate_stage_offsets()
+
+    def _calculate_stage_offsets(self) -> Dict[str, int]:
+        """计算各阶段的起始偏移量（百分比）"""
+        offsets = {}
+        offset = 0
+        for stage, weight in self.STAGE_WEIGHTS.items():
+            offsets[stage] = offset
+            offset += weight
+        return offsets
+
+    def calculate_overall_percent(self, progress: ProgressInfo) -> int:
+        """
+        计算总体进度百分比
+
+        Args:
+            progress: 进度信息
+
+        Returns:
+            总体进度百分比 (0-100)
+        """
+        stage = progress.stage
+
+        if stage == 'complete':
+            return 100
+
+        if stage not in self._stage_offsets:
+            return 0
+
+        # 计算阶段起始百分比
+        stage_offset = self._stage_offsets[stage]
+
+        # 计算阶段内的进度百分比
+        stage_percent = progress.percent
+
+        # 计算该阶段的权重
+        stage_weight = self.STAGE_WEIGHTS[stage]
+
+        # 总进度 = 阶段偏移 + (阶段内进度 * 阶段权重 / 100)
+        overall = stage_offset + int(stage_percent * stage_weight / 100)
+
+        return min(overall, 100)

gui/utils.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+GUI 工具模块
+
+提供 GUI 相关的工具类和函数。
+"""
+
+
+class RealtimeOutput:
+    """实时输出流，每次写入立即回调通知"""
+
+    def __init__(self, callback):
+        """
+        初始化实时输出流
+
+        Args:
+            callback: 写入时的回调函数，接收文本内容
+        """
+        self.callback = callback
+        self.buffer = []
+
+    def write(self, text):
+        """写入文本"""
+        if text:
+            # 将文本按行分割，逐行回调
+            lines = text.split('\n')
+            for line in lines:
+                if line:  # 忽略空行（由 split 产生）
+                    self.callback(line)
+        return len(text)
+
+    def flush(self):
+        """刷新（兼容性方法）"""
+        pass
+
+    def isatty(self):
+        """返回 False，表示不是终端"""
+        return False

utils/离散备料计划维护数据提取.py

@@ -8,12 +8,13 @@ from playwright.sync_api import sync_playwright
 from utils.excel_converter import ExcelConverter
 from utils.auth import login, logout
 from db.production_order_query import read_production_ids, query_production_order_numbers
+from typing import Callable, Optional
 
 
 class DiscreteMaterialPlanExtractor:
     """离散备料计划维护数据提取器"""
 
-    def __init__(self, username, password, headless=False, verbose=True):
+    def __init__(self, username, password, headless=False, verbose=True, progress_callback=None):
         """
         初始化提取器
 
@@ -22,11 +23,13 @@ class DiscreteMaterialPlanExtractor:
             password: 登录密码
             headless: 是否无头模式运行
             verbose: 是否打印详细日志
+            progress_callback: 进度回调函数，接收 ProgressInfo 对象
         """
         self.username = username
         self.password = password
         self.headless = headless
         self.verbose = verbose
+        self.progress_callback = progress_callback
         self.converter = ExcelConverter(verbose=verbose)
 
     def _print(self, *args, **kwargs):
@@ -34,12 +37,39 @@ class DiscreteMaterialPlanExtractor:
         if self.verbose:
             print(*args, **kwargs)
 
-    def get_production_order_numbers(self, production_id_file):
+    def _report_progress(self, stage: str, current: int, total: int, message: str, **detail):
+        """
+        报告进度
+
+        Args:
+            stage: 阶段标识
+            current: 当前进度值
+            total: 总量
+            message: 显示消息
+            **detail: 额外详细信息
+        """
+        if self.progress_callback:
+            try:
+                from gui.progress import ProgressInfo
+                progress_info = ProgressInfo(
+                    stage=stage,
+                    current=current,
+                    total=total,
+                    message=message,
+                    detail=detail
+                )
+                self.progress_callback(progress_info)
+            except Exception:
+                # 如果进度回调失败，忽略错误，不影响主流程
+                pass
+
+    def get_production_order_numbers(self, production_id_file, report_progress=False):
         """
         读取总排号文件并查询数据库获取生产订单号
 
         Args:
             production_id_file: ProductionID.txt 文件路径
+            report_progress: 是否报告进度
 
         Returns:
             生产订单号列表
@@ -52,6 +82,9 @@ class DiscreteMaterialPlanExtractor:
         order_ids = query_production_order_numbers(production_ids)
         self._print(f"查询到 {len(order_ids)} 个生产订单号")
 
+        if report_progress:
+            self._report_progress('query', 1, 1, f"查询到 {len(order_ids)} 个生产订单号", count=len(order_ids))
+
         return order_ids
 
     def group_order_ids(self, order_ids, group_size=100):
@@ -59,7 +92,7 @@ class DiscreteMaterialPlanExtractor:
         for i in range(0, len(order_ids), group_size):
             yield order_ids[i:i + group_size]
 
-    def download_batch(self, inner_frame, order_ids, batch_index, page1, debug_mode=False, debug_batch=None):
+    def download_batch(self, inner_frame, order_ids, batch_index, total_batches, page1, debug_mode=False, debug_batch=None):
         """下载一批订单号的数据"""
         from playwright.sync_api import TimeoutError
         import re
@@ -111,6 +144,15 @@ class DiscreteMaterialPlanExtractor:
         download.save_as(download_path)
         self._print(f"第 {batch_index + 1} 批下载完成: {download_path}")
 
+        # 报告进度
+        self._report_progress(
+            'download',
+            batch_index + 1,
+            total_batches,
+            f"第 {batch_index + 1}/{total_batches} 批下载完成",
+            batch_index=batch_index + 1
+        )
+
         # 关闭输出对话框（如果有的话）
         # try:
         #     inner_frame.get_by_role("button", name="取消").click()
@@ -142,6 +184,17 @@ class DiscreteMaterialPlanExtractor:
 
         for i, file_path in enumerate(file_paths, 1):
             self._print(f"转换第 {i} 个文件: {file_path}")
+
+            # 报告进度
+            self._report_progress(
+                'convert',
+                i,
+                len(file_paths),
+                f"转换第 {i}/{len(file_paths)} 个文件",
+                file_index=i,
+                file_path=file_path
+            )
+
             df = self.converter.convert(file_path, output_file=None)  # 只转换，不保存
             all_dataframes.append(df)
             self._print(f"  提取到 {len(df)} 条记录")
@@ -187,7 +240,7 @@ class DiscreteMaterialPlanExtractor:
 
     def extract(self, production_id_file, data_dir="D:/python/playwrite/data",
                output_file="D:/python/playwrite/data/离散备料计划维护_合并.xlsx",
-               debug_mode=False, debug_batch=None):
+               debug_mode=False, debug_batch=None, progress_callback=None):
         """
         执行完整的数据提取流程
 
@@ -197,76 +250,113 @@ class DiscreteMaterialPlanExtractor:
             output_file: 最终输出文件路径
             debug_mode: 是否启用调试模式
             debug_batch: 调试批次号
+            progress_callback: 进度回调函数（覆盖初始化时的回调）
 
         Returns:
             输出文件路径
         """
-        with sync_playwright() as playwright:
+        # 保存原始回调
-            # 调用登录模块
+        original_callback = self.progress_callback
-            browser, context, page, main_frame = login(
+        # 使用传入的回调或初始化时的回调
-                playwright=playwright,
+        self.progress_callback = progress_callback or self.progress_callback
-                username=self.username,
-                password=self.password,
-                headless=self.headless,
-                ignore_https_errors=True
-            )
 
-            self._print("=" * 80)
+        try:
-            self._print("开始执行离散备料计划维护数据提取")
+            with sync_playwright() as playwright:
-            self._print("=" * 80)
+                # 调用登录模块
-
+                browser, context, page, main_frame = login(
-            # 登录成功后可以进行后续操作
+                    playwright=playwright,
-            # 点击打开"功能菜单"
+                    username=self.username,
-            main_frame.locator("i").first.click()
+                    password=self.password,
-
+                    headless=self.headless,
-            # 点击打开"离散备料计划维护"
+                    ignore_https_errors=True
-            with page.expect_popup() as page1_info:
-                main_frame.get_by_title("离散备料计划维护", exact=True).first.click()
-            page1 = page1_info.value
-
-            # 获取 nested iframe
-            main_frame = page1.locator("#forwardFrame").content_frame
-            inner_frame_locator = main_frame.locator("#mainiframe")
-            inner_frame_locator.wait_for(state="visible", timeout=15000)
-            inner_frame = inner_frame_locator.content_frame
-
-            # 设置查询界面
-            self.setup_query_interface(inner_frame)
-
-            # 读取总排号并查询生产订单号
-            order_ids = self.get_production_order_numbers(production_id_file)
-
-            # 按批次下载
-            downloaded_files = []
-            for batch_index, order_ids_batch in enumerate(self.group_order_ids(order_ids, 100)):
-                self._print(f"\n=== 开始处理第 {batch_index + 1} 批，共 {len(order_ids_batch)} 个订单号 ===")
-                downloaded_file = self.download_batch(
-                    inner_frame, order_ids_batch, batch_index, page1,
-                    debug_mode=debug_mode, debug_batch=debug_batch
                 )
-                downloaded_files.append(downloaded_file)
 
-            # 执行账号注销
+                # 登录完成
-            self._print("\n开始执行账号注销...")
+                self._report_progress('login', 1, 1, "登录成功")
-            logout(main_frame, verbose=self.verbose)
 
-            # 转换并合并文件
+                self._print("=" * 80)
-            if downloaded_files:
+                self._print("开始执行离散备料计划维护数据提取")
-                self._print(f"\n=== 开始转换并合并 {len(downloaded_files)} 个文件 ===")
+                self._print("=" * 80)
-                self.convert_and_merge_files(downloaded_files, output_file)
-            else:
-                self._print("\n没有下载到任何文件")
 
-            self._print(f"\n=== 全部完成 ===")
+                # 登录成功后可以进行后续操作
-            self._print(f"最终文件: {output_file}")
+                # 点击打开"功能菜单"
+                main_frame.locator("i").first.click()
 
+                # 点击打开"离散备料计划维护"
+                with page.expect_popup() as page1_info:
+                    main_frame.get_by_title("离散备料计划维护", exact=True).first.click()
+                page1 = page1_info.value
 
+                # 获取 nested iframe
+                main_frame = page1.locator("#forwardFrame").content_frame
+                inner_frame_locator = main_frame.locator("#mainiframe")
+                inner_frame_locator.wait_for(state="visible", timeout=15000)
+                inner_frame = inner_frame_locator.content_frame
 
-            # 关闭浏览器
+                # 设置查询界面
-            context.close()
+                self.setup_query_interface(inner_frame)
-            browser.close()
 
-            return output_file
+                # 读取总排号并查询生产订单号
+                order_ids = self.get_production_order_numbers(production_id_file, report_progress=True)
+
+                # 按批次下载
+                downloaded_files = []
+                # 计算总批次数
+                total_batches = sum(1 for _ in self.group_order_ids(order_ids, 100))
+
+                for batch_index, order_ids_batch in enumerate(self.group_order_ids(order_ids, 100)):
+                    self._print(f"\n=== 开始处理第 {batch_index + 1} 批，共 {len(order_ids_batch)} 个订单号 ===")
+
+                    # 报告开始下载批次
+                    self._report_progress(
+                        'download',
+                        batch_index,
+                        total_batches,
+                        f"正在下载第 {batch_index + 1}/{total_batches} 批...",
+                        batch_index=batch_index + 1,
+                        batch_size=len(order_ids_batch)
+                    )
+
+                    downloaded_file = self.download_batch(
+                        inner_frame, order_ids_batch, batch_index, total_batches, page1,
+                        debug_mode=debug_mode, debug_batch=debug_batch
+                    )
+                    downloaded_files.append(downloaded_file)
+
+                # 执行账号注销
+                self._print("\n开始执行账号注销...")
+                self._report_progress('logout', 1, 1, "正在注销账号...")
+                logout(main_frame, verbose=self.verbose)
+
+                # 转换并合并文件
+                if downloaded_files:
+                    self._report_progress(
+                        'convert',
+                        0,
+                        len(downloaded_files),
+                        f"开始转换并合并 {len(downloaded_files)} 个文件",
+                        file_count=len(downloaded_files)
+                    )
+                    self._print(f"\n=== 开始转换并合并 {len(downloaded_files)} 个文件 ===")
+                    self.convert_and_merge_files(downloaded_files, output_file)
+                else:
+                    self._print("\n没有下载到任何文件")
+
+                self._print(f"\n=== 全部完成 ===")
+                self._print(f"最终文件: {output_file}")
+
+                # 报告完成
+                self._report_progress('complete', 1, 1, "提取完成", output_file=output_file)
+
+                # 关闭浏览器
+                context.close()
+                browser.close()
+
+                return output_file
+
+        finally:
+            # 恢复原始回调
+            self.progress_callback = original_callback
 
 def main():
     """测试函数"""