admin/BIPAuto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(extractor): Add Excel conversion and post-processing capabilities

Browse Source 
Add comprehensive post-processing features to convert downloaded Excel files
into structured data and merge them into a single output file.

New modules:
- extractor_core.py: Stateless pure functions for web operations
- excel_converter.py: Excel to DataFrame conversion utility
- tests/test_extractor_real.py: Real data extraction test suite

Enhanced features:
- post_process_downloads(): Convert and merge multiple Excel files
- extract_and_process(): Complete workflow in single call
- cleanup_temp_files(): Optional cleanup of temporary downloaded files
- Field name mapping for standardized output columns

Dependencies:
- pandas>=2.0.0 for data manipulation
- openpyxl>=3.1.0 for Excel file handling

Documentation:
- Updated CLAUDE.md with new module references
- Added API documentation for extractor components

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-03-27 14:34:56 +08:00
parent 1b984f5cfd
commit c3bbc919a5
9 changed files with 1734 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
utils/discrete_material_plan/excel_converter.py Normal file
View File

@@ -0,0 +1,280 @@
"""
Excel 报表数据转换工具组件
将 Excel 报表数据转换为数据库记录形式
"""
import pandas as pd
import openpyxl
from typing import List, Dict, Optional
import os
class ExcelConverter:
"""Excel 报表数据转换器"""
# 字段名称映射（解决字段名冲突）
FIELD_NAME_MAPPING = {"计划数量": "产品计划数量", "单位": "产品单位"}
def __init__(self, verbose: bool = True):
"""
初始化转换器
Args:
verbose: 是否打印详细日志
"""
self.verbose = verbose
def _print(self, *args, **kwargs):
"""打印日志（如果 verbose=True）"""
if self.verbose:
print(*args, **kwargs)
def convert(self, input_file: str, output_file: str = None) -> pd.DataFrame:
"""
转换 Excel 文件
Args:
input_file: 输入文件路径
output_file: 输出文件路径（可选，不指定则不保存）
Returns:
转换后的 DataFrame
"""
# 处理输出文件名
if output_file:
output_file = self._handle_output_file(output_file)
# 读取工作表
wb = openpyxl.load_workbook(input_file)
ws = wb.active
# 解析订单数据
orders = self._parse_sheet(ws)
# 转换为 DataFrame
df = self._convert_to_dataframe(orders)
if not df.empty:
# 保存文件
if output_file:
df.to_excel(output_file, index=False)
# 打印汇总报告
self._print("=" * 60)
self._print("转换完成")
self._print("=" * 60)
self._print(f"订单数: {len(orders)}")
self._print(f"数据行数: {len(df)}")
self._print(f"输出文件: {output_file if output_file else 'N/A'}")
self._print("=" * 60)
return df
def _handle_output_file(self, output_file: str) -> str:
"""
处理输出文件，如果文件存在则尝试删除
Args:
output_file: 输出文件路径
Returns:
实际使用的输出文件路径
"""
if os.path.exists(output_file):
try:
os.remove(output_file)
except PermissionError:
self._print(f"警告: 无法删除 {output_file}，可能文件被其他程序打开")
# 修改文件名
base, ext = os.path.splitext(output_file)
output_file = f"{base}_new{ext}"
return output_file
def _parse_sheet(self, ws) -> List[Dict]:
"""
解析一个工作表，返回所有订单的数据
每个订单包含:
- order_info: 订单头信息（包括页脚）
- materials: 物料数据列表
Args:
ws: openpyxl 工作表对象
Returns:
订单列表
"""
orders = []
all_rows = list(ws.iter_rows(values_only=True))
# 逐行扫描，按订单结构解析
i = 0
while i < len(all_rows):
row = all_rows[i]
# 检查是否是订单标题行
if row and "离散备料计划" in str(row[0]):
# 解析订单头信息（接下来的4行）
order_info = {}
for j in range(1, 5):
if i + j < len(all_rows) and all_rows[i + j]:
self._parse_header_row(all_rows[i + j], order_info)
# 跳过空行，找到表格标题行
table_row = i + 5
while table_row < len(all_rows) and (
not all_rows[table_row] or not all_rows[table_row][0]
):
table_row += 1
# 检查是否是表格标题行
if (
table_row < len(all_rows)
and all_rows[table_row]
and all_rows[table_row][0] == "序号"
):
# 检查表头下一行是否为空，判断是否存在数据
next_row = table_row + 1
is_empty_row = (
next_row < len(all_rows)
and all_rows[next_row]
and all(
cell is None or str(cell).strip() == ""
for cell in all_rows[next_row]
)
)
if is_empty_row:
# 没有数据，查找页脚信息
materials = []
footer_info = {}
data_row = next_row + 1
while data_row < len(all_rows) and all_rows[data_row]:
if all_rows[data_row][0] and (
"制单人" in str(all_rows[data_row][0])
or "打印人" in str(all_rows[data_row][0])
):
self._parse_header_row(all_rows[data_row], footer_info)
if (
data_row + 1 < len(all_rows)
and all_rows[data_row + 1]
):
self._parse_header_row(
all_rows[data_row + 1], footer_info
)
break
data_row += 1
orders.append(
{
"order_info": {**order_info, **footer_info},
"materials": materials,
}
)
else:
# 有数据，开始提取物料
materials = []
footer_info = {} # 页脚信息
data_row = table_row + 1
while data_row < len(all_rows) and all_rows[data_row]:
# 检查是否是页脚信息（制单人、打印人）
if all_rows[data_row + 1][0] and "制单人" in str(
all_rows[data_row + 1][0]
):
# 解析页脚信息
self._parse_header_row(all_rows[data_row], footer_info)
# 检查下一行是否也是页脚信息
if (
data_row + 1 < len(all_rows)
and all_rows[data_row + 1]
):
self._parse_header_row(
all_rows[data_row + 1], footer_info
)
break
# 提取物料数据
material_row = all_rows[data_row]
material = {
"序号": material_row[0],
"材料编码": material_row[1],
"材料名称": material_row[2],
"规格": material_row[3],
"型号": material_row[4],
"图号": material_row[5],
"物料材质": material_row[6],
"计划数量": material_row[7],
"单位": material_row[8],
"需用日期": material_row[9],
"发料仓库": material_row[10],
"单位用量": material_row[11],
"累计出库数量": material_row[12],
}
materials.append(material)
data_row += 1
orders.append(
{
"order_info": {**order_info, **footer_info},
"materials": materials,
}
)
i += 1
return orders
def _parse_header_row(self, row: tuple, info: Dict):
"""
解析订单头信息的一行（字段名和值交错排列）
Args:
row: 行数据
info: 存储解析结果的字典
"""
i = 0
while i < len(row):
cell = row[i]
if cell and str(cell).strip() and "：" in str(cell):
# 找到字段名
field_name = str(cell).replace("：", "").strip()
# 应用字段名映射
if field_name in self.FIELD_NAME_MAPPING:
field_name = self.FIELD_NAME_MAPPING[field_name]
# 跳过空单元格，找到第一个非字段名的值
j = i + 1
while j < len(row) and (
not row[j] or not str(row[j]).strip() or "：" in str(row[j])
):
j += 1
if j < len(row) and row[j] and not "：" in str(row[j]):
info[field_name] = str(row[j]).strip()
# 跳过已处理的值，继续找下一个字段名
i = j + 1
else:
i += 1
def _convert_to_dataframe(self, orders: List[Dict]) -> pd.DataFrame:
"""
将订单数据转换为扁平化的 DataFrame
Args:
orders: 订单列表
Returns:
扁平化的 DataFrame
"""
all_records = []
for order in orders:
order_info = order["order_info"]
materials = order["materials"]
for material in materials:
record = {**order_info, **material}
all_records.append(record)
return pd.DataFrame(all_records)

45
utils/discrete_material_plan/extractor.py
View File

@@ -144,6 +144,7 @@ def post_process_downloads(
downloaded_files: List[str],
output_file: str,
verbose: bool = True,
cleanup_temp_files: bool = True,
) -> Tuple[str, pd.DataFrame]:
"""
Convert and merge downloaded Excel files into structured DataFrame.
@@ -154,6 +155,7 @@ def post_process_downloads(
downloaded_files: List of paths to downloaded Excel files
output_file: Path to save merged Excel result
verbose: Whether to print progress messages
cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True)
Returns:
Tuple of (output_file_path, merged_dataframe)
@@ -162,7 +164,8 @@ def post_process_downloads(
>>> output_path, df = post_process_downloads(
... downloaded_files=["batch_1.xlsx", "batch_2.xlsx"],
... output_file="merged.xlsx",
... verbose=True
... verbose=True,
... cleanup_temp_files=True
... )
"""
from .excel_converter import ExcelConverter
@@ -194,9 +197,37 @@ def post_process_downloads(
print(f"Merged result saved to: {output_path}")
print(f"Total rows: {len(merged_df)}")
# Cleanup temporary downloaded files
if cleanup_temp_files:
_cleanup_temp_files(downloaded_files, verbose)
return str(output_path), merged_df
def _cleanup_temp_files(downloaded_files: List[str], verbose: bool = True) -> int:
"""
Remove temporary downloaded files.
Args:
downloaded_files: List of file paths to delete
verbose: Whether to print progress messages
Returns:
Number of files successfully deleted
"""
deleted_count = 0
for file_path in downloaded_files:
try:
Path(file_path).unlink()
deleted_count += 1
if verbose:
print(f"Deleted temp file: {file_path}")
except Exception as e:
if verbose:
print(f"Warning: Could not delete {file_path}: {e}")
return deleted_count
def extract_and_post_process(
work_frame: Frame,
page: Page,
@@ -205,6 +236,7 @@ def extract_and_post_process(
output_file: str,
batch_size: int = 10,
verbose: bool = True,
cleanup_temp_files: bool = True,
) -> Tuple[str, pd.DataFrame]:
"""
Complete extraction workflow: download batches + post-process to merged Excel.
@@ -220,6 +252,7 @@ def extract_and_post_process(
output_file: Path for final merged Excel output
batch_size: Maximum order IDs per batch
verbose: Whether to print progress messages
cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True)
Returns:
Tuple of (output_file_path, merged_dataframe)
@@ -229,7 +262,8 @@ def extract_and_post_process(
>>> browser, context, page, main_frame = login(...)
>>> work_frame, page1 = navigate_to_discrete_material_page(main_frame, page)
>>> output_path, df = extract_and_post_process(
... work_frame, page1, order_ids, "/downloads", "output.xlsx"
... work_frame, page1, order_ids, "/downloads", "output.xlsx",
... cleanup_temp_files=True
... )
>>> context.close()
>>> browser.close()
@@ -254,6 +288,7 @@ def extract_and_post_process(
downloaded_files=downloaded_files,
output_file=output_file,
verbose=verbose,
cleanup_temp_files=cleanup_temp_files,
)
return output_path, merged_df
@@ -292,6 +327,7 @@ def extract_from_file(
output_file: str,
batch_size: int = 10,
verbose: bool = True,
cleanup_temp_files: bool = True,
) -> Tuple[str, pd.DataFrame]:
"""
Extract data from order IDs in a file and post-process to merged Excel.
@@ -306,6 +342,7 @@ def extract_from_file(
output_file: Path for final merged Excel output
batch_size: Maximum order IDs per batch
verbose: Whether to print progress messages
cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True)
Returns:
Tuple of (output_file_path, merged_dataframe)
@@ -315,7 +352,8 @@ def extract_from_file(
>>> order_ids = read_order_ids_from_file("orders.txt")
>>> # Extract and process
>>> output_path, df = extract_from_file(
... "orders.txt", work_frame, page, "/downloads", "output.xlsx"
... "orders.txt", work_frame, page, "/downloads", "output.xlsx",
... cleanup_temp_files=True
... )
"""
order_ids = read_order_ids_from_file(id_file)
@@ -331,4 +369,5 @@ def extract_from_file(
output_file=output_file,
batch_size=batch_size,
verbose=verbose,
cleanup_temp_files=cleanup_temp_files,
)

164
utils/discrete_material_plan/extractor_core.py Normal file
View File

@@ -0,0 +1,164 @@
"""
Core Web Operations Module
Pure functions for Yonyou BIP discrete material plan maintenance page interactions.
All functions are stateless and accept required parameters explicitly.
"""
import re
import os
from typing import List
from playwright.sync_api import Page, Frame, TimeoutError
def navigate_to_discrete_material_page(main_frame: Frame, page: Page) -> tuple[Frame, Page]:
"""
Navigate to the discrete material plan maintenance page.
Args:
main_frame: The main forwardFrame iframe
page: The Playwright page object
Returns:
tuple: (work_frame, page1) - The work iframe and the new popup page
"""
# Click icon to open menu
main_frame.locator("i").first.click()
# Wait for popup and click menu item
with page.expect_popup() as page1_info:
main_frame.get_by_title("离散备料计划维护", exact=True).first.click()
page1 = page1_info.value
# Get nested iframes
f_frame = page1.locator("#forwardFrame").content_frame
inner_frame_locator = f_frame.locator("#mainiframe")
inner_frame_locator.wait_for(state="visible", timeout=15000)
work_frame = inner_frame_locator.content_frame
return work_frame, page1
def setup_query_interface(work_frame: Frame) -> None:
"""
Initialize the query interface by selecting order number query tab.
Args:
work_frame: The inner work iframe containing the query interface
"""
# Open search panel
work_frame.locator(".search-name-wrapper > .iconfont").click()
# Select order number query
work_frame.get_by_text("订单号查询").click()
# Select "All" tab
work_frame.get_by_role("tab", name="全部").click()
# Set page size to 5000
input_box = work_frame.locator("#rc_select_0")
input_box.fill("5000")
input_box.press("Enter")
def fill_and_search_orders(work_frame: Frame, order_ids: List[str]) -> None:
"""
Fill order IDs into the search textbox and trigger search.
Args:
work_frame: The work iframe containing the search form
order_ids: List of order IDs to search for
"""
textbox = work_frame.get_by_role("textbox", name="来源生产订单号")
# Clear and fill order IDs
textbox.fill("")
textbox.fill(",".join(order_ids))
# Click search button
work_frame.locator(".search-component-searchBtn").click()
# Wait for loading to complete
loading_locator = work_frame.locator("div").filter(has_text="加载中").nth(1)
try:
loading_locator.wait_for(state="visible", timeout=3000)
loading_locator.wait_for(state="hidden", timeout=0)
except TimeoutError:
pass
def download_batch_data(
work_frame: Frame,
page: Page,
order_ids: List[str],
batch_index: int,
download_dir: str
) -> str:
"""
Execute the download workflow for a single batch of order IDs.
Args:
work_frame: The work iframe containing the data grid
page: The Playwright page object for download handling
order_ids: List of order IDs to download
batch_index: Zero-based batch index for naming the output file
download_dir: Directory path to save the downloaded file
Returns:
str: Full path to the downloaded file
Raises:
TimeoutError: If UI elements are not found or operations timeout
"""
# Step 1: Select first row
work_frame.get_by_role("row", name="序号").get_by_label("").click()
# Step 2: Hover over "More" button
work_frame.get_by_role("button", name="更多").hover()
# Step 3: Click "Export"
work_frame.get_by_text("输出", exact=True).click()
# Step 4: Set row threshold
threshold_box = (
work_frame.locator("div")
.filter(has_text=re.compile(r"^行数阈值$"))
.locator("input[type='text']")
)
threshold_box.fill("300000")
# Step 5: Trigger download and save file
download_filename = f"temp_batch_{batch_index + 1}.xlsx"
download_path = os.path.join(download_dir, download_filename)
with page.expect_download() as download_info:
work_frame.get_by_role("button", name="确定(Y)").click()
download = download_info.value
download.save_as(download_path)
return download_path
def execute_batch_download_workflow(
work_frame: Frame,
page: Page,
order_ids: List[str],
batch_index: int,
download_dir: str
) -> str:
"""
Complete workflow: fill orders, search, and download for a single batch.
Args:
work_frame: The work iframe containing the search form and data grid
page: The Playwright page object for download handling
order_ids: List of order IDs for this batch
batch_index: Zero-based batch index for naming the output file
download_dir: Directory path to save the downloaded file
Returns:
str: Full path to the downloaded file
"""
fill_and_search_orders(work_frame, order_ids)
return download_batch_data(work_frame, page, order_ids, batch_index, download_dir)