admin/playwrite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor download and file merging logic to use ExcelConverter for improved data handling

Browse Source
This commit is contained in:
Misaka_Company
2026-01-16 13:01:54 +08:00
parent 4d441be304
commit c8a9c33ecc
1 changed files with 24 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
main.py
View File

@@ -7,6 +7,8 @@ import re
import os import os
import pandas as pd import pandas as pd
from time import sleep from time import sleep
from utils.excel_converter import ExcelConverter
def read_order_ids(file_path): def read_order_ids(file_path):
"""读取订单号文件""" """读取订单号文件"""
@@ -15,11 +17,13 @@ def read_order_ids(file_path):
order_ids = [line.strip() for line in f if line.strip()] order_ids = [line.strip() for line in f if line.strip()]
return order_ids return order_ids
def group_order_ids(order_ids, group_size=100): def group_order_ids(order_ids, group_size=100):
"""将订单号分组""" """将订单号分组"""
for i in range(0, len(order_ids), group_size): for i in range(0, len(order_ids), group_size):
yield order_ids[i:i + group_size] yield order_ids[i:i + group_size]
def download_batch(inner_frame, order_ids, batch_index, page1, debug_mode=False, debug_batch=None): def download_batch(inner_frame, order_ids, batch_index, page1, debug_mode=False, debug_batch=None):
"""下载一批订单号的数据""" """下载一批订单号的数据"""
# 清空文本框 # 清空文本框
@@ -32,7 +36,7 @@ def download_batch(inner_frame, order_ids, batch_index, page1, debug_mode=False,
# 点击查询 # 点击查询
inner_frame.locator(".search-component-searchBtn").click() inner_frame.locator(".search-component-searchBtn").click()
print(f"第 {batch_index + 1} 批查询完成，等待加载结果...") print(f"第 {batch_index + 1} 批查询完成，等待加载结果...")
#sleep(3)
# 等待加载完成 # 等待加载完成
loading_locator = inner_frame.locator("div").filter(has_text="加载中").nth(1) loading_locator = inner_frame.locator("div").filter(has_text="加载中").nth(1)
try: try:
@@ -80,19 +84,23 @@ def download_batch(inner_frame, order_ids, batch_index, page1, debug_mode=False,
return download_path return download_path
def merge_excel_files(file_paths, output_path):
"""合并多个 Excel 文件""" def convert_and_merge_files(file_paths, output_path):
"""使用 ExcelConverter 转换并合并所有文件"""
converter = ExcelConverter(verbose=False) # 不打印详细日志
all_dataframes = [] all_dataframes = []
for file_path in file_paths: for i, file_path in enumerate(file_paths, 1):
df = pd.read_excel(file_path) print(f"转换第 {i} 个文件: {file_path}")
df = converter.convert(file_path, output_file=None) # 只转换，不保存
all_dataframes.append(df) all_dataframes.append(df)
print(f"已读取: {file_path}, 共 {len(df)} 行") print(f" 提取到 {len(df)} 条记录")
if all_dataframes: if all_dataframes:
print(f"\n合并 {len(all_dataframes)} 个文件的数据...")
merged_df = pd.concat(all_dataframes, ignore_index=True) merged_df = pd.concat(all_dataframes, ignore_index=True)
merged_df.to_excel(output_path, index=False) merged_df.to_excel(output_path, index=False)
print(f"合并完成: {output_path}, 总共 {len(merged_df)} 行") print(f"合并完成: {output_path}, 总共 {len(merged_df)} 条记录")
# 删除临时文件 # 删除临时文件
for file_path in file_paths: for file_path in file_paths:
@@ -102,6 +110,7 @@ def merge_excel_files(file_paths, output_path):
return output_path return output_path
return None return None
def main(): def main():
with sync_playwright() as playwright: with sync_playwright() as playwright:
# 调用登录模块 # 调用登录模块
@@ -132,7 +141,7 @@ def main():
inner_frame.locator(".search-name-wrapper > .iconfont").click() inner_frame.locator(".search-name-wrapper > .iconfont").click()
inner_frame.get_by_text("订单号查询").click() inner_frame.get_by_text("订单号查询").click()
inner_frame.get_by_role("tab", name="全部").click() inner_frame.get_by_role("tab", name="全部").click()
# 填充并验证，如果失败则重试 # 填充并验证，如果失败则重试
max_retries = 3 max_retries = 3
expected_value = "5000" expected_value = "5000"
@@ -162,20 +171,16 @@ def main():
downloaded_files = [] downloaded_files = []
for batch_index, order_ids_batch in enumerate(group_order_ids(order_ids, 100)): for batch_index, order_ids_batch in enumerate(group_order_ids(order_ids, 100)):
print(f"\n=== 开始处理第 {batch_index + 1} 批，共 {len(order_ids_batch)} 个订单号 ===") print(f"\n=== 开始处理第 {batch_index + 1} 批，共 {len(order_ids_batch)} 个订单号 ===")
downloaded_file = download_batch(inner_frame, order_ids_batch, batch_index, page1, debug_mode=False, debug_batch=1) downloaded_file = download_batch(inner_frame, order_ids_batch, batch_index, page1, debug_mode=False, debug_batch=None)
downloaded_files.append(downloaded_file) downloaded_files.append(downloaded_file)
# 合并文件 # 转换并合并文件
if len(downloaded_files) > 1: if downloaded_files:
print(f"\n=== 开始合并 {len(downloaded_files)} 个文件 ===") print(f"\n=== 开始转换并合并 {len(downloaded_files)} 个文件 ===")
final_output = "D:/python/playwrite/data/离散备料计划维护_合并.xlsx" final_output = "D:/python/playwrite/data/离散备料计划维护_合并.xlsx"
merge_excel_files(downloaded_files, final_output) convert_and_merge_files(downloaded_files, final_output)
else: else:
print("\n只下载了一个文件，无需合并") print("\n没有下载到任何文件")
final_output = downloaded_files[0]
# 重命名为最终文件名
os.rename(final_output, "D:/python/playwrite/data/离散备料计划维护_合并.xlsx")
final_output = "D:/python/playwrite/data/离散备料计划维护_合并.xlsx"
print(f"\n=== 全部完成 ===") print(f"\n=== 全部完成 ===")
print(f"最终文件: {final_output}") print(f"最终文件: {final_output}")
@@ -186,5 +191,6 @@ def main():
context.close() context.close()
browser.close() browser.close()
if __name__ == "__main__": if __name__ == "__main__":
main() main()