feat(extractor): Add Excel conversion and post-processing capabilities
Add comprehensive post-processing features to convert downloaded Excel files into structured data and merge them into a single output file. New modules: - extractor_core.py: Stateless pure functions for web operations - excel_converter.py: Excel to DataFrame conversion utility - tests/test_extractor_real.py: Real data extraction test suite Enhanced features: - post_process_downloads(): Convert and merge multiple Excel files - extract_and_process(): Complete workflow in single call - cleanup_temp_files(): Optional cleanup of temporary downloaded files - Field name mapping for standardized output columns Dependencies: - pandas>=2.0.0 for data manipulation - openpyxl>=3.1.0 for Excel file handling Documentation: - Updated CLAUDE.md with new module references - Added API documentation for extractor components Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
11
utils/__init__.py
Normal file
11
utils/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""
|
||||
Utils package for BIPAuto automation framework.
|
||||
"""
|
||||
|
||||
from .auth import login, logout
|
||||
|
||||
__all__ = [
|
||||
# Auth module
|
||||
"login",
|
||||
"logout",
|
||||
]
|
||||
280
utils/discrete_material_plan/excel_converter.py
Normal file
280
utils/discrete_material_plan/excel_converter.py
Normal file
@@ -0,0 +1,280 @@
|
||||
"""
|
||||
Excel 报表数据转换工具组件
|
||||
将 Excel 报表数据转换为数据库记录形式
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import openpyxl
|
||||
from typing import List, Dict, Optional
|
||||
import os
|
||||
|
||||
|
||||
class ExcelConverter:
|
||||
"""Excel 报表数据转换器"""
|
||||
|
||||
# 字段名称映射（解决字段名冲突）
|
||||
FIELD_NAME_MAPPING = {"计划数量": "产品计划数量", "单位": "产品单位"}
|
||||
|
||||
def __init__(self, verbose: bool = True):
|
||||
"""
|
||||
初始化转换器
|
||||
|
||||
Args:
|
||||
verbose: 是否打印详细日志
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def _print(self, *args, **kwargs):
|
||||
"""打印日志（如果 verbose=True）"""
|
||||
if self.verbose:
|
||||
print(*args, **kwargs)
|
||||
|
||||
def convert(self, input_file: str, output_file: str = None) -> pd.DataFrame:
|
||||
"""
|
||||
转换 Excel 文件
|
||||
|
||||
Args:
|
||||
input_file: 输入文件路径
|
||||
output_file: 输出文件路径（可选，不指定则不保存）
|
||||
|
||||
Returns:
|
||||
转换后的 DataFrame
|
||||
"""
|
||||
# 处理输出文件名
|
||||
if output_file:
|
||||
output_file = self._handle_output_file(output_file)
|
||||
|
||||
# 读取工作表
|
||||
wb = openpyxl.load_workbook(input_file)
|
||||
ws = wb.active
|
||||
|
||||
# 解析订单数据
|
||||
orders = self._parse_sheet(ws)
|
||||
|
||||
# 转换为 DataFrame
|
||||
df = self._convert_to_dataframe(orders)
|
||||
|
||||
if not df.empty:
|
||||
# 保存文件
|
||||
if output_file:
|
||||
df.to_excel(output_file, index=False)
|
||||
|
||||
# 打印汇总报告
|
||||
self._print("=" * 60)
|
||||
self._print("转换完成")
|
||||
self._print("=" * 60)
|
||||
self._print(f"订单数: {len(orders)}")
|
||||
self._print(f"数据行数: {len(df)}")
|
||||
self._print(f"输出文件: {output_file if output_file else 'N/A'}")
|
||||
self._print("=" * 60)
|
||||
|
||||
return df
|
||||
|
||||
def _handle_output_file(self, output_file: str) -> str:
|
||||
"""
|
||||
处理输出文件，如果文件存在则尝试删除
|
||||
|
||||
Args:
|
||||
output_file: 输出文件路径
|
||||
|
||||
Returns:
|
||||
实际使用的输出文件路径
|
||||
"""
|
||||
if os.path.exists(output_file):
|
||||
try:
|
||||
os.remove(output_file)
|
||||
except PermissionError:
|
||||
self._print(f"警告: 无法删除 {output_file}，可能文件被其他程序打开")
|
||||
# 修改文件名
|
||||
base, ext = os.path.splitext(output_file)
|
||||
output_file = f"{base}_new{ext}"
|
||||
return output_file
|
||||
|
||||
def _parse_sheet(self, ws) -> List[Dict]:
|
||||
"""
|
||||
解析一个工作表，返回所有订单的数据
|
||||
|
||||
每个订单包含:
|
||||
- order_info: 订单头信息（包括页脚）
|
||||
- materials: 物料数据列表
|
||||
|
||||
Args:
|
||||
ws: openpyxl 工作表对象
|
||||
|
||||
Returns:
|
||||
订单列表
|
||||
"""
|
||||
orders = []
|
||||
all_rows = list(ws.iter_rows(values_only=True))
|
||||
|
||||
# 逐行扫描，按订单结构解析
|
||||
i = 0
|
||||
while i < len(all_rows):
|
||||
row = all_rows[i]
|
||||
|
||||
# 检查是否是订单标题行
|
||||
if row and "离散备料计划" in str(row[0]):
|
||||
# 解析订单头信息（接下来的4行）
|
||||
order_info = {}
|
||||
for j in range(1, 5):
|
||||
if i + j < len(all_rows) and all_rows[i + j]:
|
||||
self._parse_header_row(all_rows[i + j], order_info)
|
||||
|
||||
# 跳过空行，找到表格标题行
|
||||
table_row = i + 5
|
||||
while table_row < len(all_rows) and (
|
||||
not all_rows[table_row] or not all_rows[table_row][0]
|
||||
):
|
||||
table_row += 1
|
||||
|
||||
# 检查是否是表格标题行
|
||||
if (
|
||||
table_row < len(all_rows)
|
||||
and all_rows[table_row]
|
||||
and all_rows[table_row][0] == "序号"
|
||||
):
|
||||
# 检查表头下一行是否为空，判断是否存在数据
|
||||
next_row = table_row + 1
|
||||
is_empty_row = (
|
||||
next_row < len(all_rows)
|
||||
and all_rows[next_row]
|
||||
and all(
|
||||
cell is None or str(cell).strip() == ""
|
||||
for cell in all_rows[next_row]
|
||||
)
|
||||
)
|
||||
|
||||
if is_empty_row:
|
||||
# 没有数据，查找页脚信息
|
||||
materials = []
|
||||
footer_info = {}
|
||||
data_row = next_row + 1
|
||||
while data_row < len(all_rows) and all_rows[data_row]:
|
||||
if all_rows[data_row][0] and (
|
||||
"制单人" in str(all_rows[data_row][0])
|
||||
or "打印人" in str(all_rows[data_row][0])
|
||||
):
|
||||
self._parse_header_row(all_rows[data_row], footer_info)
|
||||
if (
|
||||
data_row + 1 < len(all_rows)
|
||||
and all_rows[data_row + 1]
|
||||
):
|
||||
self._parse_header_row(
|
||||
all_rows[data_row + 1], footer_info
|
||||
)
|
||||
break
|
||||
data_row += 1
|
||||
|
||||
orders.append(
|
||||
{
|
||||
"order_info": {**order_info, **footer_info},
|
||||
"materials": materials,
|
||||
}
|
||||
)
|
||||
else:
|
||||
# 有数据，开始提取物料
|
||||
materials = []
|
||||
footer_info = {} # 页脚信息
|
||||
data_row = table_row + 1
|
||||
while data_row < len(all_rows) and all_rows[data_row]:
|
||||
# 检查是否是页脚信息（制单人、打印人）
|
||||
if all_rows[data_row + 1][0] and "制单人" in str(
|
||||
all_rows[data_row + 1][0]
|
||||
):
|
||||
# 解析页脚信息
|
||||
self._parse_header_row(all_rows[data_row], footer_info)
|
||||
# 检查下一行是否也是页脚信息
|
||||
if (
|
||||
data_row + 1 < len(all_rows)
|
||||
and all_rows[data_row + 1]
|
||||
):
|
||||
self._parse_header_row(
|
||||
all_rows[data_row + 1], footer_info
|
||||
)
|
||||
break
|
||||
|
||||
# 提取物料数据
|
||||
material_row = all_rows[data_row]
|
||||
material = {
|
||||
"序号": material_row[0],
|
||||
"材料编码": material_row[1],
|
||||
"材料名称": material_row[2],
|
||||
"规格": material_row[3],
|
||||
"型号": material_row[4],
|
||||
"图号": material_row[5],
|
||||
"物料材质": material_row[6],
|
||||
"计划数量": material_row[7],
|
||||
"单位": material_row[8],
|
||||
"需用日期": material_row[9],
|
||||
"发料仓库": material_row[10],
|
||||
"单位用量": material_row[11],
|
||||
"累计出库数量": material_row[12],
|
||||
}
|
||||
materials.append(material)
|
||||
|
||||
data_row += 1
|
||||
|
||||
orders.append(
|
||||
{
|
||||
"order_info": {**order_info, **footer_info},
|
||||
"materials": materials,
|
||||
}
|
||||
)
|
||||
|
||||
i += 1
|
||||
|
||||
return orders
|
||||
|
||||
def _parse_header_row(self, row: tuple, info: Dict):
|
||||
"""
|
||||
解析订单头信息的一行（字段名和值交错排列）
|
||||
|
||||
Args:
|
||||
row: 行数据
|
||||
info: 存储解析结果的字典
|
||||
"""
|
||||
i = 0
|
||||
while i < len(row):
|
||||
cell = row[i]
|
||||
if cell and str(cell).strip() and "：" in str(cell):
|
||||
# 找到字段名
|
||||
field_name = str(cell).replace("：", "").strip()
|
||||
|
||||
# 应用字段名映射
|
||||
if field_name in self.FIELD_NAME_MAPPING:
|
||||
field_name = self.FIELD_NAME_MAPPING[field_name]
|
||||
|
||||
# 跳过空单元格，找到第一个非字段名的值
|
||||
j = i + 1
|
||||
while j < len(row) and (
|
||||
not row[j] or not str(row[j]).strip() or "：" in str(row[j])
|
||||
):
|
||||
j += 1
|
||||
if j < len(row) and row[j] and not "：" in str(row[j]):
|
||||
info[field_name] = str(row[j]).strip()
|
||||
# 跳过已处理的值，继续找下一个字段名
|
||||
i = j + 1
|
||||
else:
|
||||
i += 1
|
||||
|
||||
def _convert_to_dataframe(self, orders: List[Dict]) -> pd.DataFrame:
|
||||
"""
|
||||
将订单数据转换为扁平化的 DataFrame
|
||||
|
||||
Args:
|
||||
orders: 订单列表
|
||||
|
||||
Returns:
|
||||
扁平化的 DataFrame
|
||||
"""
|
||||
all_records = []
|
||||
|
||||
for order in orders:
|
||||
order_info = order["order_info"]
|
||||
materials = order["materials"]
|
||||
|
||||
for material in materials:
|
||||
record = {**order_info, **material}
|
||||
all_records.append(record)
|
||||
|
||||
return pd.DataFrame(all_records)
|
||||
@@ -144,6 +144,7 @@ def post_process_downloads(
|
||||
downloaded_files: List[str],
|
||||
output_file: str,
|
||||
verbose: bool = True,
|
||||
cleanup_temp_files: bool = True,
|
||||
) -> Tuple[str, pd.DataFrame]:
|
||||
"""
|
||||
Convert and merge downloaded Excel files into structured DataFrame.
|
||||
@@ -154,6 +155,7 @@ def post_process_downloads(
|
||||
downloaded_files: List of paths to downloaded Excel files
|
||||
output_file: Path to save merged Excel result
|
||||
verbose: Whether to print progress messages
|
||||
cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True)
|
||||
|
||||
Returns:
|
||||
Tuple of (output_file_path, merged_dataframe)
|
||||
@@ -162,7 +164,8 @@ def post_process_downloads(
|
||||
>>> output_path, df = post_process_downloads(
|
||||
... downloaded_files=["batch_1.xlsx", "batch_2.xlsx"],
|
||||
... output_file="merged.xlsx",
|
||||
... verbose=True
|
||||
... verbose=True,
|
||||
... cleanup_temp_files=True
|
||||
... )
|
||||
"""
|
||||
from .excel_converter import ExcelConverter
|
||||
@@ -194,9 +197,37 @@ def post_process_downloads(
|
||||
print(f"Merged result saved to: {output_path}")
|
||||
print(f"Total rows: {len(merged_df)}")
|
||||
|
||||
# Cleanup temporary downloaded files
|
||||
if cleanup_temp_files:
|
||||
_cleanup_temp_files(downloaded_files, verbose)
|
||||
|
||||
return str(output_path), merged_df
|
||||
|
||||
|
||||
def _cleanup_temp_files(downloaded_files: List[str], verbose: bool = True) -> int:
|
||||
"""
|
||||
Remove temporary downloaded files.
|
||||
|
||||
Args:
|
||||
downloaded_files: List of file paths to delete
|
||||
verbose: Whether to print progress messages
|
||||
|
||||
Returns:
|
||||
Number of files successfully deleted
|
||||
"""
|
||||
deleted_count = 0
|
||||
for file_path in downloaded_files:
|
||||
try:
|
||||
Path(file_path).unlink()
|
||||
deleted_count += 1
|
||||
if verbose:
|
||||
print(f"Deleted temp file: {file_path}")
|
||||
except Exception as e:
|
||||
if verbose:
|
||||
print(f"Warning: Could not delete {file_path}: {e}")
|
||||
return deleted_count
|
||||
|
||||
|
||||
def extract_and_post_process(
|
||||
work_frame: Frame,
|
||||
page: Page,
|
||||
@@ -205,6 +236,7 @@ def extract_and_post_process(
|
||||
output_file: str,
|
||||
batch_size: int = 10,
|
||||
verbose: bool = True,
|
||||
cleanup_temp_files: bool = True,
|
||||
) -> Tuple[str, pd.DataFrame]:
|
||||
"""
|
||||
Complete extraction workflow: download batches + post-process to merged Excel.
|
||||
@@ -220,6 +252,7 @@ def extract_and_post_process(
|
||||
output_file: Path for final merged Excel output
|
||||
batch_size: Maximum order IDs per batch
|
||||
verbose: Whether to print progress messages
|
||||
cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True)
|
||||
|
||||
Returns:
|
||||
Tuple of (output_file_path, merged_dataframe)
|
||||
@@ -229,7 +262,8 @@ def extract_and_post_process(
|
||||
>>> browser, context, page, main_frame = login(...)
|
||||
>>> work_frame, page1 = navigate_to_discrete_material_page(main_frame, page)
|
||||
>>> output_path, df = extract_and_post_process(
|
||||
... work_frame, page1, order_ids, "/downloads", "output.xlsx"
|
||||
... work_frame, page1, order_ids, "/downloads", "output.xlsx",
|
||||
... cleanup_temp_files=True
|
||||
... )
|
||||
>>> context.close()
|
||||
>>> browser.close()
|
||||
@@ -254,6 +288,7 @@ def extract_and_post_process(
|
||||
downloaded_files=downloaded_files,
|
||||
output_file=output_file,
|
||||
verbose=verbose,
|
||||
cleanup_temp_files=cleanup_temp_files,
|
||||
)
|
||||
|
||||
return output_path, merged_df
|
||||
@@ -292,6 +327,7 @@ def extract_from_file(
|
||||
output_file: str,
|
||||
batch_size: int = 10,
|
||||
verbose: bool = True,
|
||||
cleanup_temp_files: bool = True,
|
||||
) -> Tuple[str, pd.DataFrame]:
|
||||
"""
|
||||
Extract data from order IDs in a file and post-process to merged Excel.
|
||||
@@ -306,6 +342,7 @@ def extract_from_file(
|
||||
output_file: Path for final merged Excel output
|
||||
batch_size: Maximum order IDs per batch
|
||||
verbose: Whether to print progress messages
|
||||
cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True)
|
||||
|
||||
Returns:
|
||||
Tuple of (output_file_path, merged_dataframe)
|
||||
@@ -315,7 +352,8 @@ def extract_from_file(
|
||||
>>> order_ids = read_order_ids_from_file("orders.txt")
|
||||
>>> # Extract and process
|
||||
>>> output_path, df = extract_from_file(
|
||||
... "orders.txt", work_frame, page, "/downloads", "output.xlsx"
|
||||
... "orders.txt", work_frame, page, "/downloads", "output.xlsx",
|
||||
... cleanup_temp_files=True
|
||||
... )
|
||||
"""
|
||||
order_ids = read_order_ids_from_file(id_file)
|
||||
@@ -331,4 +369,5 @@ def extract_from_file(
|
||||
output_file=output_file,
|
||||
batch_size=batch_size,
|
||||
verbose=verbose,
|
||||
cleanup_temp_files=cleanup_temp_files,
|
||||
)
|
||||
|
||||
164
utils/discrete_material_plan/extractor_core.py
Normal file
164
utils/discrete_material_plan/extractor_core.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Core Web Operations Module
|
||||
Pure functions for Yonyou BIP discrete material plan maintenance page interactions.
|
||||
All functions are stateless and accept required parameters explicitly.
|
||||
"""
|
||||
|
||||
import re
|
||||
import os
|
||||
from typing import List
|
||||
from playwright.sync_api import Page, Frame, TimeoutError
|
||||
|
||||
|
||||
def navigate_to_discrete_material_page(main_frame: Frame, page: Page) -> tuple[Frame, Page]:
|
||||
"""
|
||||
Navigate to the discrete material plan maintenance page.
|
||||
|
||||
Args:
|
||||
main_frame: The main forwardFrame iframe
|
||||
page: The Playwright page object
|
||||
|
||||
Returns:
|
||||
tuple: (work_frame, page1) - The work iframe and the new popup page
|
||||
"""
|
||||
# Click icon to open menu
|
||||
main_frame.locator("i").first.click()
|
||||
|
||||
# Wait for popup and click menu item
|
||||
with page.expect_popup() as page1_info:
|
||||
main_frame.get_by_title("离散备料计划维护", exact=True).first.click()
|
||||
|
||||
page1 = page1_info.value
|
||||
|
||||
# Get nested iframes
|
||||
f_frame = page1.locator("#forwardFrame").content_frame
|
||||
inner_frame_locator = f_frame.locator("#mainiframe")
|
||||
inner_frame_locator.wait_for(state="visible", timeout=15000)
|
||||
work_frame = inner_frame_locator.content_frame
|
||||
|
||||
return work_frame, page1
|
||||
|
||||
|
||||
def setup_query_interface(work_frame: Frame) -> None:
|
||||
"""
|
||||
Initialize the query interface by selecting order number query tab.
|
||||
|
||||
Args:
|
||||
work_frame: The inner work iframe containing the query interface
|
||||
"""
|
||||
# Open search panel
|
||||
work_frame.locator(".search-name-wrapper > .iconfont").click()
|
||||
|
||||
# Select order number query
|
||||
work_frame.get_by_text("订单号查询").click()
|
||||
|
||||
# Select "All" tab
|
||||
work_frame.get_by_role("tab", name="全部").click()
|
||||
|
||||
# Set page size to 5000
|
||||
input_box = work_frame.locator("#rc_select_0")
|
||||
input_box.fill("5000")
|
||||
input_box.press("Enter")
|
||||
|
||||
|
||||
def fill_and_search_orders(work_frame: Frame, order_ids: List[str]) -> None:
|
||||
"""
|
||||
Fill order IDs into the search textbox and trigger search.
|
||||
|
||||
Args:
|
||||
work_frame: The work iframe containing the search form
|
||||
order_ids: List of order IDs to search for
|
||||
"""
|
||||
textbox = work_frame.get_by_role("textbox", name="来源生产订单号")
|
||||
|
||||
# Clear and fill order IDs
|
||||
textbox.fill("")
|
||||
textbox.fill(",".join(order_ids))
|
||||
|
||||
# Click search button
|
||||
work_frame.locator(".search-component-searchBtn").click()
|
||||
|
||||
# Wait for loading to complete
|
||||
loading_locator = work_frame.locator("div").filter(has_text="加载中").nth(1)
|
||||
try:
|
||||
loading_locator.wait_for(state="visible", timeout=3000)
|
||||
loading_locator.wait_for(state="hidden", timeout=0)
|
||||
except TimeoutError:
|
||||
pass
|
||||
|
||||
|
||||
def download_batch_data(
|
||||
work_frame: Frame,
|
||||
page: Page,
|
||||
order_ids: List[str],
|
||||
batch_index: int,
|
||||
download_dir: str
|
||||
) -> str:
|
||||
"""
|
||||
Execute the download workflow for a single batch of order IDs.
|
||||
|
||||
Args:
|
||||
work_frame: The work iframe containing the data grid
|
||||
page: The Playwright page object for download handling
|
||||
order_ids: List of order IDs to download
|
||||
batch_index: Zero-based batch index for naming the output file
|
||||
download_dir: Directory path to save the downloaded file
|
||||
|
||||
Returns:
|
||||
str: Full path to the downloaded file
|
||||
|
||||
Raises:
|
||||
TimeoutError: If UI elements are not found or operations timeout
|
||||
"""
|
||||
# Step 1: Select first row
|
||||
work_frame.get_by_role("row", name="序号").get_by_label("").click()
|
||||
|
||||
# Step 2: Hover over "More" button
|
||||
work_frame.get_by_role("button", name="更多").hover()
|
||||
|
||||
# Step 3: Click "Export"
|
||||
work_frame.get_by_text("输出", exact=True).click()
|
||||
|
||||
# Step 4: Set row threshold
|
||||
threshold_box = (
|
||||
work_frame.locator("div")
|
||||
.filter(has_text=re.compile(r"^行数阈值$"))
|
||||
.locator("input[type='text']")
|
||||
)
|
||||
threshold_box.fill("300000")
|
||||
|
||||
# Step 5: Trigger download and save file
|
||||
download_filename = f"temp_batch_{batch_index + 1}.xlsx"
|
||||
download_path = os.path.join(download_dir, download_filename)
|
||||
|
||||
with page.expect_download() as download_info:
|
||||
work_frame.get_by_role("button", name="确定(Y)").click()
|
||||
|
||||
download = download_info.value
|
||||
download.save_as(download_path)
|
||||
|
||||
return download_path
|
||||
|
||||
|
||||
def execute_batch_download_workflow(
|
||||
work_frame: Frame,
|
||||
page: Page,
|
||||
order_ids: List[str],
|
||||
batch_index: int,
|
||||
download_dir: str
|
||||
) -> str:
|
||||
"""
|
||||
Complete workflow: fill orders, search, and download for a single batch.
|
||||
|
||||
Args:
|
||||
work_frame: The work iframe containing the search form and data grid
|
||||
page: The Playwright page object for download handling
|
||||
order_ids: List of order IDs for this batch
|
||||
batch_index: Zero-based batch index for naming the output file
|
||||
download_dir: Directory path to save the downloaded file
|
||||
|
||||
Returns:
|
||||
str: Full path to the downloaded file
|
||||
"""
|
||||
fill_and_search_orders(work_frame, order_ids)
|
||||
return download_batch_data(work_frame, page, order_ids, batch_index, download_dir)
|
||||
Reference in New Issue
Block a user