From c3bbc919a5801054249e614ac3ba2558dff33770 Mon Sep 17 00:00:00 2001 From: Misaka_Company Date: Fri, 27 Mar 2026 14:34:56 +0800 Subject: [PATCH] feat(extractor): Add Excel conversion and post-processing capabilities Add comprehensive post-processing features to convert downloaded Excel files into structured data and merge them into a single output file. New modules: - extractor_core.py: Stateless pure functions for web operations - excel_converter.py: Excel to DataFrame conversion utility - tests/test_extractor_real.py: Real data extraction test suite Enhanced features: - post_process_downloads(): Convert and merge multiple Excel files - extract_and_process(): Complete workflow in single call - cleanup_temp_files(): Optional cleanup of temporary downloaded files - Field name mapping for standardized output columns Dependencies: - pandas>=2.0.0 for data manipulation - openpyxl>=3.1.0 for Excel file handling Documentation: - Updated CLAUDE.md with new module references - Added API documentation for extractor components Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 5 + CLAUDE.md | 65 ++ docs/discrete_material_plan_extractor_api.md | 999 ++++++++++++++++++ requirements.txt | 2 + tests/test_extractor_real.py | 166 +++ utils/__init__.py | 11 + .../discrete_material_plan/excel_converter.py | 280 +++++ utils/discrete_material_plan/extractor.py | 45 +- .../discrete_material_plan/extractor_core.py | 164 +++ 9 files changed, 1734 insertions(+), 3 deletions(-) create mode 100644 docs/discrete_material_plan_extractor_api.md create mode 100644 tests/test_extractor_real.py create mode 100644 utils/__init__.py create mode 100644 utils/discrete_material_plan/excel_converter.py create mode 100644 utils/discrete_material_plan/extractor_core.py diff --git a/.gitignore b/.gitignore index f4c2c02..3e2224e 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,8 @@ Thumbs.db .agents/ .agent/ .claude/ +.sisyphus + +data/ + +nul \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index a4b0a54..ddb9ffa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,34 @@ Note: Test scripts are responsible for loading environment variables and passing - Returns tuple: `(browser, context, page, main_frame)` where `main_frame` is the forwardFrame iframe. - Callers are responsible for browser lifecycle management (context.close(), browser.close()). +**`utils/discrete_material_plan/extractor_core.py`** - Core web operations module (pure functions) +- `navigate_to_discrete_material_page(main_frame, page)` - Navigate to discrete material plan maintenance page. Returns `(work_frame, page1)`. +- `setup_query_interface(work_frame)` - Initialize query interface by selecting order number query tab and setting page size. +- `fill_and_search_orders(work_frame, order_ids)` - Fill order IDs into search textbox and trigger search. +- `download_batch_data(work_frame, page, order_ids, batch_index, download_dir)` - Execute download workflow for a single batch. Returns downloaded file path. +- `execute_batch_download_workflow(work_frame, page, order_ids, batch_index, download_dir)` - Complete workflow: fill orders, search, and download. Returns downloaded file path. +- All functions are stateless and accept required parameters explicitly. No logging or progress reporting - callers handle that. +- **See**: `docs/discrete_material_plan_extractor_core.md` for complete API documentation and usage examples. + +**`utils/discrete_material_plan/extractor.py`** - High-level extractor component +- `DiscreteMaterialPlanExtractor` class - Batch processing wrapper with progress reporting and error handling. +- `extract_from_file()` function - Convenience function to extract data from order IDs in a file. +- **New Methods**: + - `post_process_downloads()` - Convert and merge downloaded Excel files into structured format + - `extract_and_process()` - Complete workflow: extract data and post-process in one call +- Accepts ID list and file paths as parameters, handles session management, batch processing, Excel conversion, and file merging. +- **Documentation**: + - Basic Usage: `docs/discrete_material_plan_extractor.md` - Basic API and usage examples + - Component Guide: `docs/extractor_component_guide.md` - Complete guide with Mermaid diagrams + - Post-Processing: `docs/extractor_post_processing.md` - Excel conversion and merging features + +**`utils/discrete_material_plan/excel_converter.py`** - Excel data conversion utility +- `ExcelConverter` class - Converts raw Excel reports to structured DataFrames. +- `convert(input_file, output_file)` - Converts Excel file and returns DataFrame. +- Handles nested order structures, flattens to table format, applies field name mapping. +- Used internally by extractor for post-processing, can also be used standalone. +- **See**: `docs/extractor_post_processing.md` for usage examples. + **`tests/`** - Test suite - All test files must add `PROJECT_ROOT` to `sys.path` to import `utils` modules - Tests follow pattern: load dotenv, set browser path environment variable, run Playwright operations @@ -72,6 +100,13 @@ Callers must construct the complete login URL before passing to `login()`: url = f"{os.getenv('ERP_URL').rstrip('/')}/yonbip/resources/uap/rbac/login/main/index.html" ``` +### Demo Data + +**`id-demo.txt`** - Demo production IDs for testing +- Contains 15 sample production IDs (SC70202603240xxx format) +- Used by test scripts and demos for web operations testing +- Format: One ID per line, plain text file + ## Common Commands ```bash @@ -81,10 +116,40 @@ python tests/test_auth_config.py # Run login/logout test python tests/test_login.py +# Run web operations tests +python tests/test_web_operations.py + +# Run extractor component tests +python tests/test_extractor_component.py + +# Run real data extraction test (uses tests/id-demo.txt) +python tests/test_extractor_real.py + +# Or use quick run scripts +./run_extractor_test.bat # Windows +./run_extractor_test.sh # Linux/Mac + +# Run web operations demo +python tests/demo_web_operations.py + +# Run extractor component demo +python demo_extractor_component.py + # Run any test with virtual environment source .venv/Scripts/activate && python tests/.py ``` +## Documentation + +**`docs/INDEX.md`** - Complete documentation index +- Quick start guide, API reference, examples, and troubleshooting +- Central hub for all documentation + +**`docs/extractor_component_guide.md`** - Complete component guide with Mermaid diagrams +- Architecture overview, class structure, workflow diagrams +- State management, error handling, integration examples +- Visual documentation using Mermaid graphs + ## Code Conventions 1. **English Only**: All user-facing output, docstrings, comments, and log messages must be in English. Chinese text is only used for Playwright element selectors matching the actual UI. diff --git a/docs/discrete_material_plan_extractor_api.md b/docs/discrete_material_plan_extractor_api.md new file mode 100644 index 0000000..e996ccf --- /dev/null +++ b/docs/discrete_material_plan_extractor_api.md @@ -0,0 +1,999 @@ +# Discrete Material Plan Extractor - API Reference + +> **Pure Function Data Extraction for Yonyou BIP** +> +> Stateless functions for extracting discrete material plan data from Yonyou BIP ERP system with batch processing and Excel post-processing capabilities. + +--- + +## Overview + +The `extractor` module provides high-level pure functions for extracting material plan data from the Yonyou BIP ERP system. All functions are **stateless** and **explicitly accept required parameters**, following the same design patterns as `extractor_core.py` and `auth.py`. + +### Architecture Overview + +```mermaid +graph TB + subgraph User["User Code"] + A[Main Script] + end + + subgraph Auth["utils/auth.py"] + B[login] + C[logout] + end + + subgraph Extractor["utils/discrete_material_plan/extractor.py"] + D[extract_and_post_process] + E[extract_batches] + F[post_process_downloads] + G[read_order_ids_from_file] + H[chunk_order_ids] + end + + subgraph Core["utils/discrete_material_plan/extractor_core.py"] + I[navigate_to_discrete_material_page] + J[setup_query_interface] + K[execute_batch_download_workflow] + end + + subgraph Converter["utils/discrete_material_plan/excel_converter.py"] + L[ExcelConverter.convert] + end + + subgraph External["External"] + M[(Yonyou BIP ERP)] + N[(Downloaded Excel Files)] + O[(Merged Output Excel)] + end + + A --> B + A --> D + A --> G + + B --> M + D --> E + D --> F + E --> H + E --> K + F --> L + + B --> I + I --> M + K --> M + K --> N + + L --> N + L --> O + + style User fill:#e1f5ff + style Auth fill:#fff4e1 + style Extractor fill:#e8f5e9 + style Core fill:#fce4ec + style Converter fill:#f3e5f5 + style External fill:#ffebee +``` + +**Key Features:** +- ✅ Stateless pure functions (no class instances) +- ✅ Explicit parameter passing +- ✅ Batch processing for large order lists +- ✅ Excel conversion and merging +- ✅ Progress reporting with verbose logging +- ✅ Caller-managed session lifecycle + +**Module Location:** `utils/discrete_material_plan/extractor.py` + +--- + +## Quick Start + +### Basic Usage - Complete Workflow + +```python +from utils.discrete_material_plan import ( + extract_and_post_process, + get_login_url, +) +from utils.auth import login +from utils.discrete_material_plan import navigate_to_discrete_material_page +from playwright.sync_api import sync_playwright + +# 1. Setup browser session +with sync_playwright() as playwright: + browser, context, page, main_frame = login( + playwright=playwright, + username="your_username", + password="your_password", + url=get_login_url("https://erp.example.com"), + headless=True, + ignore_https_errors=True, + ) + + try: + # 2. Navigate to discrete material page + work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) + + # 3. Extract and process data + output_path, df = extract_and_post_process( + work_frame=work_frame, + page=page1, + order_ids=["SC70202603240001", "SC70202603240002"], + download_dir="./downloads", + output_file="./output/merged_data.xlsx", + batch_size=10, + verbose=True, + ) + + print(f"Extracted {len(df)} records to {output_path}") + + finally: + # 4. Caller manages browser lifecycle + context.close() + browser.close() +``` + +### From File - Simplest Approach + +```python +from utils.discrete_material_plan import read_order_ids_from_file + +# Read order IDs from file +order_ids = read_order_ids_from_file("order_ids.txt") +print(f"Loaded {len(order_ids)} order IDs") +``` + +--- + +## API Reference + +### Helper Functions + +#### `chunk_order_ids(order_ids: List[str], batch_size: int) -> List[List[str]]` + +Split order IDs into batches for processing. + +**Parameters:** +| Name | Type | Description | +|------|------|-------------| +| `order_ids` | `List[str]` | List of order IDs to process | +| `batch_size` | `int` | Maximum order IDs per batch | + +**Returns:** `List[List[str]]` - List of batches + +**Example:** +```python +>>> chunk_order_ids(["A", "B", "C", "D", "E"], 2) +[["A", "B"], ["C", "D"], ["E"]] +``` + +--- + +#### `get_login_url(base_url: str) -> str` + +Construct complete login URL from base ERP URL. + +**Parameters:** +| Name | Type | Description | +|------|------|-------------| +| `base_url` | `str` | Base ERP URL (e.g., "https://erp.example.com") | + +**Returns:** `str` - Complete login page URL + +**Example:** +```python +>>> get_login_url("https://erp.example.com") +"https://erp.example.com/yonbip/resources/uap/rbac/login/main/index.html" +``` + +--- + +#### `read_order_ids_from_file(id_file: str, encoding: str = "utf-8") -> List[str]` + +Read order IDs from a text file (one ID per line). + +**Parameters:** +| Name | Type | Default | Description | +|------|------|---------|-------------| +| `id_file` | `str` | - | Path to file containing order IDs | +| `encoding` | `str` | `"utf-8"` | File encoding | + +**Returns:** `List[str]` - List of order IDs (empty lines filtered) + +**Raises:** +- `FileNotFoundError` - If id_file doesn't exist + +**Example:** +```python +>>> order_ids = read_order_ids_from_file("orders.txt") +>>> len(order_ids) +15 +``` + +--- + +### Core Extraction Functions + +#### `extract_batch(work_frame, page, order_ids, batch_index, download_dir) -> str` + +Execute download workflow for a single batch of order IDs. + +**Parameters:** +| Name | Type | Description | +|------|------|-------------| +| `work_frame` | `Frame` | Work iframe containing the data grid | +| `page` | `Page` | Playwright page object for download handling | +| `order_ids` | `List[str]` | Order IDs for this batch | +| `batch_index` | `int` | Zero-based batch index (for file naming) | +| `download_dir` | `str` | Directory to save downloaded file | + +**Returns:** `str` - Path to downloaded Excel file + +**Note:** This is a thin wrapper around `execute_batch_download_workflow` from `extractor_core.py`. + +--- + +#### `extract_batches(work_frame, page, order_ids, download_dir, batch_size=10) -> List[str]` + +Download data for multiple batches of order IDs. + +**Parameters:** +| Name | Type | Default | Description | +|------|------|---------|-------------| +| `work_frame` | `Frame` | Work iframe containing the data grid | +| `page` | `Page` | Playwright page object | +| `order_ids` | `List[str]` | List of order IDs to download | +| `download_dir` | `str` | Directory to save downloaded files | +| `batch_size` | `int` | `10` | Max order IDs per batch | + +**Returns:** `List[str]` - List of downloaded file paths + +**Example:** +```python +# Caller manages browser session +browser, context, page, main_frame = login(...) +work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) +setup_query_interface(work_frame) + +files = extract_batches( + work_frame=work_frame, + page=page1, + order_ids=["ID1", "ID2", "ID3"], + download_dir="./downloads", + batch_size=5 +) + +context.close() +browser.close() +``` + +--- + +### High-Level Workflow Functions + +#### `extract_and_post_process(work_frame, page, order_ids, download_dir, output_file, batch_size=10, verbose=True) -> Tuple[str, DataFrame]` + +Complete extraction workflow: download batches + post-process to merged Excel. + +### Workflow Diagram + +```mermaid +sequenceDiagram + participant User as User Code + participant Auth as utils.auth.login + participant Nav as navigate_to_discrete
_material_page + participant Setup as setup_query
_interface + participant Extract as extract_and_post
_process + participant Batches as extract_batches + participant PostProcess as post_process
_downloads + participant Converter as ExcelConverter + participant Browser as Browser + participant ERP as Yonyou BIP ERP + participant FS as File System + + User->>Auth: login(credentials) + Auth->>Browser: Launch + Auth->>ERP: Authenticate + Auth-->>User: (browser, context, page, main_frame) + + User->>Nav: navigate(main_frame, page) + Nav->>ERP: Load page + Nav-->>User: (work_frame, page1) + + User->>Setup: setup_query_interface(work_frame) + Setup->>ERP: Configure query panel + + User->>Extract: extract_and_post_process(params) + + Note over Extract,PostProcess: Phase 1: Download + Extract->>Batches: extract_batches(work_frame, page, order_ids) + + loop For each batch + Batches->>Batches: chunk_order_ids(order_ids, batch_size) + Batches->>Setup: setup_query_interface() + Batches->>ERP: Fill order IDs + Search + Batches->>ERP: Click Export + ERP-->>FS: Save batch_N.xlsx + Batches-->>Extract: [batch_1.xlsx, batch_2.xlsx, ...] + end + + Note over Extract,PostProcess: Phase 2: Post-Process + Extract->>PostProcess: post_process_downloads(files, output_file) + + loop For each downloaded file + PostProcess->>Converter: convert(batch_N.xlsx) + Converter->>FS: Read Excel + Converter->>Converter: Parse nested structure + Converter-->>PostProcess: DataFrame + end + + PostProcess->>PostProcess: pd.concat(all_dfs) + PostProcess->>FS: Write merged.xlsx + PostProcess-->>Extract: (output_path, merged_df) + + Extract-->>User: (output_path, DataFrame) + + User->>Browser: context.close() + User->>Browser: browser.close() +``` + +**Parameters:** +| Name | Type | Default | Description | +|------|------|---------|-------------| +| `work_frame` | `Frame` | - | Work iframe containing the data grid | +| `page` | `Page` | - | Playwright page object for download handling | +| `order_ids` | `List[str]` | - | List of order IDs to extract | +| `download_dir` | `str` | - | Directory for temporary batch files | +| `output_file` | `str` | - | Path for final merged Excel output | +| `batch_size` | `int` | `10` | Max order IDs per batch | +| `verbose` | `bool` | `True` | Print progress messages | + +**Returns:** `Tuple[str, pd.DataFrame]` - (output_file_path, merged_dataframe) + +**Example:** +```python +from utils.discrete_material_plan import extract_and_post_process + +# Caller manages session (see full example above) +output_path, df = extract_and_post_process( + work_frame=work_frame, + page=page1, + order_ids=["SC70202603240001", "SC70202603240002"], + download_dir="./downloads", + output_file="./output/merged.xlsx", + batch_size=10, + verbose=True, +) + +print(f"Saved {len(df)} records to {output_path}") +``` + +**Workflow:** +1. Downloads order data in batches to `download_dir` +2. Converts each downloaded Excel file to DataFrame +3. Merges all DataFrames into one +4. Saves merged result to `output_file` +5. Returns (path, dataframe) tuple + +--- + +#### `extract_from_file(id_file, work_frame, page, download_dir, output_file, batch_size=10, verbose=True) -> Tuple[str, DataFrame]` + +Extract data from order IDs in a file and post-process to merged Excel. + +**Parameters:** +| Name | Type | Default | Description | +|------|------|---------|-------------| +| `id_file` | `str` | - | Path to file with order IDs (one per line) | +| `work_frame` | `Frame` | Work iframe containing the data grid | +| `page` | `Page` | Playwright page object for download | +| `download_dir` | `str` | Directory for temporary batch files | +| `output_file` | `str` | Path for final merged Excel output | +| `batch_size` | `int` | `10` | Max order IDs per batch | +| `verbose` | `bool` | `True` | Print progress messages | + +**Returns:** `Tuple[str, pd.DataFrame]` - (output_file_path, merged_dataframe) + +**Example:** +```python +from utils.discrete_material_plan import extract_from_file + +# After setting up browser session and navigating to page +output_path, df = extract_from_file( + id_file="order_ids.txt", + work_frame=work_frame, + page=page1, + download_dir="./downloads", + output_file="./output/results.xlsx", + batch_size=10, + verbose=True, +) +``` + +**File Format:** +``` +SC70202603240001 +SC70202603240002 +SC70202603240003 +``` + +--- + +### Post-Processing Functions + +#### `post_process_downloads(downloaded_files, output_file, verbose=True) -> Tuple[str, DataFrame]` + +Convert and merge downloaded Excel files into structured DataFrame. + +**Parameters:** +| Name | Type | Default | Description | +|------|------|---------|-------------| +| `downloaded_files` | `List[str]` | - | List of downloaded Excel file paths | +| `output_file` | `str` | - | Path to save merged Excel result | +| `verbose` | `bool` | `True` | Print progress messages | + +**Returns:** `Tuple[str, pd.DataFrame]` - (output_file_path, merged_dataframe) + +**Example:** +```python +from utils.discrete_material_plan import post_process_downloads + +output_path, df = post_process_downloads( + downloaded_files=[ + "./downloads/batch_1.xlsx", + "./downloads/batch_2.xlsx", + ], + output_file="./output/merged.xlsx", + verbose=True, +) + +print(f"Merged {len(df)} records") +``` + +**Process:** +1. Uses `ExcelConverter` to convert each file +2. Merges all DataFrames with `pd.concat()` +3. Saves merged result to `output_file` + +--- + +## Architecture + +### Function Hierarchy + +```mermaid +graph LR + subgraph Level1["Level 1: Entry Points"] + A1[extract_and_post_process] + A2[extract_from_file] + end + + subgraph Level2["Level 2: Workflow Orchestration"] + B1[extract_batches] + B2[post_process_downloads] + B3[read_order_ids_from_file] + end + + subgraph Level3["Level 3: Core Operations"] + C1[extract_batch] + C2[chunk_order_ids] + C3[setup_query_interface] + C4[ExcelConverter.convert] + end + + subgraph Level4["Level 4: Low-Level"] + D1[execute_batch_download_workflow] + D2[fill_and_search_orders] + D3[download_batch_data] + end + + A1 --> B1 + A1 --> B2 + A2 --> B3 + A2 --> A1 + + B1 --> C1 + B1 --> C2 + B1 --> C3 + B2 --> C4 + + C1 --> D1 + C3 --> D2 + D1 --> D2 + D1 --> D3 + + style Level1 fill:#e3f2fd + style Level2 fill:#fff3e0 + style Level3 fill:#f3e5f5 + style Level4 fill:#e8f5e9 +``` + +### Design Principles + +**1. Stateless Pure Functions** +```python +# ✅ Correct: Stateless, explicit parameters +output_path, df = extract_and_post_process( + work_frame=work_frame, + page=page, + order_ids=order_ids, + # ... explicit params +) + +# ❌ Wrong: Stateful class (OLD approach - removed) +# extractor = DiscreteMaterialPlanExtractor(username, password, ...) +``` + +**2. Caller-Managed Lifecycle** +```python +# Caller manages browser session +browser, context, page, main_frame = login(...) +try: + work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) + result = extract_and_post_process(work_frame, page1, order_ids, ...) +finally: + context.close() # Caller closes + browser.close() # Caller closes +``` + +**3. Explicit Dependencies** +```python +# ❌ No implicit state +def extract(order_ids): # Missing required params + ... + +# ✅ All params explicit +def extract(work_frame, page, order_ids, download_dir, output_file): + ... +``` + +### Module Dependencies + +```mermaid +graph TD + subgraph Extractor["extractor.py"] + E1[extract_and_post_process] + E2[extract_batches] + E3[post_process_downloads] + end + + subgraph Core["extractor_core.py"] + C1[execute_batch_download_workflow] + C2[setup_query_interface] + C3[fill_and_search_orders] + C4[download_batch_data] + end + + subgraph Converter["excel_converter.py"] + K1[ExcelConverter] + K2[convert] + end + + subgraph External["External Libraries"] + P1[pandas DataFrame] + P2[openpyxl] + end + + E1 --> E2 + E1 --> E3 + E2 --> C1 + E2 --> C2 + E3 --> K1 + K1 --> K2 + K2 --> P1 + K1 --> P2 + + C1 --> C2 + C1 --> C3 + C1 --> C4 + + style Extractor fill:#e8f5e9 + style Core fill:#fff3e0 + style Converter fill:#f3e5f5 + style External fill:#e3f2fd +``` + +**Low Coupling:** +- No direct dependency on `utils.auth` +- Session objects (`work_frame`, `page`) passed as parameters +- Caller controls lifecycle + +--- + +## Integration Patterns + +### Data Flow Diagram + +```mermaid +flowchart LR + subgraph Input["Input"] + I1[Order IDs
List or File] + end + + subgraph Download["Download Phase
Requires Browser"] + D1[Chunk into
Batches] + D2[Search Orders
in ERP] + D3[Export to
Excel] + D4[(Batch Excel
Files)] + end + + subgraph Process["Post-Process Phase
No Browser Needed"] + P1[Read Excel
Files] + P2[Parse Nested
Structure] + P3[Flatten to
DataFrame] + P4[Concatenate
All Batches] + P5[(Merged
Excel)] + end + + subgraph Output["Output"] + O1[DataFrame
Object] + O2[Excel File] + end + + I1 --> D1 + D1 --> D2 + D2 --> D3 + D3 --> D4 + D4 --> P1 + P1 --> P2 + P2 --> P3 + P3 --> P4 + P4 --> P5 + P4 --> O1 + P5 --> O2 + + style Input fill:#e1f5ff + style Download fill:#fff4e1 + style Process fill:#e8f5e9 + style Output fill:#fce4ec +``` + +--- + +## Integration Patterns + +### Pattern 1: Complete Workflow with Session Management + +```python +from playwright.sync_api import sync_playwright +from utils.auth import login, get_login_url +from utils.discrete_material_plan import ( + navigate_to_discrete_material_page, + extract_and_post_process, +) + +def main(): + with sync_playwright() as playwright: + # Setup session + browser, context, page, main_frame = login( + playwright=playwright, + username="admin", + password="secret", + url=get_login_url("https://erp.example.com"), + headless=True, + ) + + try: + # Navigate + work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) + + # Extract + output_path, df = extract_and_post_process( + work_frame=work_frame, + page=page1, + order_ids=["ID1", "ID2", "ID3"], + download_dir="./downloads", + output_file="./output/result.xlsx", + ) + + print(f"Done: {len(df)} records") + + finally: + # Cleanup + context.close() + browser.close() +``` + +### Pattern 2: Two-Phase (Download Then Process) + +```python +from utils.discrete_material_plan import ( + extract_batches, + post_process_downloads, + setup_query_interface, + navigate_to_discrete_material_page, +) +from utils.auth import login + +# Phase 1: Download +browser, context, page, main_frame = login(...) +work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) + +downloaded_files = extract_batches( + work_frame=work_frame, + page=page1, + order_ids=order_ids, + download_dir="./downloads", + batch_size=10, +) + +context.close() +browser.close() + +# Phase 2: Process (can be done later, even without browser) +output_path, df = post_process_downloads( + downloaded_files=downloaded_files, + output_file="./output/merged.xlsx", +) +``` + +### Pattern 3: Custom Batch Processing with Error Handling + +```python +from utils.discrete_material_plan import chunk_order_ids, extract_batch +from utils.auth import login + +order_ids = [...] # Large list +batches = chunk_order_ids(order_ids, batch_size=10) + +success_files = [] +failed_batches = [] + +for i, batch in enumerate(batches): + try: + file_path = extract_batch( + work_frame=work_frame, + page=page, + order_ids=batch, + batch_index=i, + download_dir="./downloads", + ) + success_files.append(file_path) + print(f"Batch {i+1}/{len(batches)} OK") + except Exception as e: + failed_batches.append(i) + print(f"Batch {i+1} failed: {e}") + +print(f"Success: {len(success_files)}, Failed: {len(failed_batches)}") +``` + +--- + +## Testing + +### Unit Tests + +```bash +# Run component tests +source .venv/Scripts/activate && python tests/test_extractor_component.py +``` + +**Test Coverage:** +- ✅ `chunk_order_ids()` - Batch splitting logic +- ✅ `get_login_url()` - URL construction +- ✅ `read_order_ids_from_file()` - File reading +- ✅ Module exports verification + +### Integration Tests + +```bash +# Run real data extraction test +source .venv/Scripts/activate && python tests/test_extractor_real.py +``` + +**Note:** Integration tests require: +- Valid ERP credentials in `.env` +- Playwright browser installed +- Network access to ERP system + +--- + +## Error Handling + +### Common Errors + +**FileNotFoundError:** +```python +>>> read_order_ids_from_file("nonexistent.txt") +FileNotFoundError: Order ID file not found: nonexistent.txt +``` + +**Timeout during extraction:** +```python +try: + output_path, df = extract_and_post_process(...) +except TimeoutError as e: + print(f"Operation timed out: {e}") + # Browser session may need re-initialization +``` + +**Permission errors (file access):** +```python +try: + post_process_downloads(downloaded_files, output_file) +except PermissionError: + print(f"Cannot write to {output_file} - file may be open in another program") +``` + +### Best Practices + +```python +# 1. Always cleanup browser sessions +try: + # extraction logic +finally: + context.close() + browser.close() + +# 2. Validate order IDs before extraction +order_ids = read_order_ids_from_file("orders.txt") +if not order_ids: + raise ValueError("No order IDs to process") + +# 3. Ensure output directory exists +from pathlib import Path +Path("./output").mkdir(parents=True, exist_ok=True) +``` + +--- + +## Performance Considerations + +### Batch Size Tuning + +**Small batches (5-10):** +- ✅ More frequent progress updates +- ✅ Easier to recover from failures +- ❌ More UI interactions (slower) + +**Large batches (50-100):** +- ✅ Faster overall (fewer UI interactions) +- ✅ Better for large datasets +- ❌ Single failure affects more records + +**Recommendation:** Start with `batch_size=10`, adjust based on: +- Total order count +- Network stability +- UI response time + +### Memory Usage + +For very large extractions (1000+ orders): +```python +# Process in stages to limit memory +all_dfs = [] +for batch_files in batch_groups: + _, df = post_process_downloads(batch_files, output_file) + all_dfs.append(df) + +# Final merge +merged_df = pd.concat(all_dfs, ignore_index=True) +``` + +--- + +## Migration Guide + +### From Old Class-Based API (v1) to New Function API (v2) + +### API Comparison + +```mermaid +mindmap + root((API Comparison)) + v1 Old API + Class-Based + ::icon(fa fa-times-circle) + Stateful + Internal session management + Hidden dependencies + Usage + extractor = DiscreteMaterialPlanExtractor(...) + extractor.extract_from_file(...) + Issues + Hard to test + Tight coupling + Implicit state + v2 New API + Pure Functions + ::icon(fa fa-check-circle) + Stateless + Caller-managed session + Explicit dependencies + Usage + extract_and_post_process(params...) + Return: (output_path, DataFrame) + Benefits + Easy to test + Low coupling + Clear data flow + +``` + +**Old (v1):** +```python +from utils.discrete_material_plan import DiscreteMaterialPlanExtractor + +extractor = DiscreteMaterialPlanExtractor( + username="admin", + password="secret", + base_url="https://erp.example.com", +) + +output_path = extractor.extract_from_file("orders.txt", "output.xlsx") +``` + +**New (v2):** +```python +from utils.discrete_material_plan import ( + extract_from_file, + get_login_url, + navigate_to_discrete_material_page, +) +from utils.auth import login + +# Caller manages session +browser, context, page, main_frame = login( + playwright, + username="admin", + password="secret", + url=get_login_url("https://erp.example.com"), +) + +try: + work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) + + output_path, df = extract_from_file( + id_file="orders.txt", + work_frame=work_frame, + page=page1, + download_dir="./downloads", + output_file="./output/result.xlsx", + ) +finally: + context.close() + browser.close() +``` + +### Version Comparison Table + +| Aspect | v1 (Class-Based) | v2 (Pure Functions) | +|--------|------------------|---------------------| +| **Pattern** | `DiscreteMaterialPlanExtractor` class | Standalone functions | +| **State** | Instance variables (`self.username`, etc.) | No state, explicit params | +| **Session** | Internal management | Caller manages | +| **Return** | `str` (output path only) | `Tuple[str, DataFrame]` | +| **Testing** | Hard (requires mocking class) | Easy (pure functions) | +| **Coupling** | High (depends on `utils.auth`) | Low (session passed as param) | +| **Flexibility** | Limited (fixed workflow) | High (can use individual functions) | + +**Key Changes:** +1. Class removed → pure functions +2. Session management moved to caller +3. Return value now includes DataFrame tuple +4. All dependencies explicit + +--- + +## Related Documentation + +- [`extractor_core.py`](discrete_material_plan_extractor_core.md) - Low-level web operations +- [`excel_converter.py`](extractor_post_processing.md) - Excel conversion +- [`auth.py`](authentication.md) - Authentication and session management + +--- + +## Version + +**Current:** v2.0.0 (stateless pure functions) + +**Breaking Changes in v2:** +- Removed `DiscreteMaterialPlanExtractor` class +- Changed to caller-managed session lifecycle +- All functions now accept explicit parameters + +--- + +## License + +Part of BIPAuto project - see project root for license information. diff --git a/requirements.txt b/requirements.txt index 0d2473f..c4d3ff4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ playwright>=1.40.0 python-dotenv>=1.0.0 +pandas>=2.0.0 +openpyxl>=3.1.0 diff --git a/tests/test_extractor_real.py b/tests/test_extractor_real.py new file mode 100644 index 0000000..09edfa2 --- /dev/null +++ b/tests/test_extractor_real.py @@ -0,0 +1,166 @@ +""" +Integration test for discrete_material_plan.extractor module + +Tests the complete extraction workflow using real ERP data. +Requires valid ERP credentials in .env file. + +This test: +1. Reads order IDs from tests/id-demo.txt +2. Downloads data from Yonyou BIP ERP system +3. Saves raw Excel files to data/downloads/ +4. Converts and merges to data/output/merged_result.xlsx +""" + +import sys +from pathlib import Path +from datetime import datetime + +# Add project root to Python path +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +# Load environment variables +from dotenv import load_dotenv +load_dotenv(PROJECT_ROOT / ".env") + +# Configure browser path +import os +os.environ["PLAYWRIGHT_BROWSERS_PATH"] = os.getenv("PLAYWRIGHT_BROWSERS_PATH", "") + +# Generate timestamp for unique output files +TIMESTAMP = datetime.now().strftime("%Y%m%d_%H%M%S") + +# Output paths +DOWNLOAD_DIR = PROJECT_ROOT / "data" / "downloads" +OUTPUT_DIR = PROJECT_ROOT / "data" / "output" +OUTPUT_FILE = OUTPUT_DIR / f"merged_result_{TIMESTAMP}.xlsx" + +# Ensure directories exist +DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True) +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + +print("=" * 60) +print("Discrete Material Plan Extractor - Integration Test") +print("=" * 60) +print(f"\nConfiguration:") +print(f" ID File: {PROJECT_ROOT / 'tests' / 'id-demo.txt'}") +print(f" Download Dir: {DOWNLOAD_DIR}") +print(f" Output File: {OUTPUT_FILE}") +print(f" ERP URL: {os.getenv('ERP_URL', 'Not set')}") +print(f" Headless: {os.getenv('ERP_HEADLESS', 'true')}") +print("=" * 60) + +# Verify environment variables +required_vars = ["ERP_USERNAME", "ERP_PASSWORD", "ERP_URL"] +missing_vars = [var for var in required_vars if not os.getenv(var)] + +if missing_vars: + print(f"\n[ERROR] Missing required environment variables: {missing_vars}") + print("Please ensure .env file contains:") + print(" ERP_USERNAME=your_username") + print(" ERP_PASSWORD=your_password") + print(" ERP_URL=https://your-erp-url.com") + sys.exit(1) + +try: + # Import required modules + print("\n[1/6] Importing modules...") + from playwright.sync_api import sync_playwright + from utils.auth import login + from utils.discrete_material_plan import ( + navigate_to_discrete_material_page, + extract_and_post_process, + read_order_ids_from_file, + get_login_url, + ) + print("[OK] Modules imported successfully") + + # Read order IDs from demo file + print("\n[2/6] Reading order IDs from data/demo/id-demo.txt...") + id_file = PROJECT_ROOT / "data" / "demo" / "id-demo.txt" + order_ids = read_order_ids_from_file(str(id_file)) + print(f"[OK] Loaded {len(order_ids)} order IDs") + + test_order_ids = order_ids + + # Setup browser and extract data + print("\n[3/6] Launching browser and logging in...") + with sync_playwright() as playwright: + # Login + browser, context, page, main_frame = login( + playwright=playwright, + username=os.getenv("ERP_USERNAME"), + password=os.getenv("ERP_PASSWORD"), + url=get_login_url(os.getenv("ERP_URL")), + headless=os.getenv("ERP_HEADLESS", "true").lower() == "true", + ignore_https_errors=os.getenv("ERP_IGNORE_HTTPS_ERRORS", "true").lower() == "true", + verbose=True, + ) + + try: + # Navigate to discrete material page + print("\n[4/6] Navigating to discrete material plan page...") + work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) + print("[OK] Navigation successful") + + # Extract and post-process + print("\n[5/6] Extracting data and post-processing...") + print(f" Orders: {len(test_order_ids)}") + print(f" Batch size: 100") + print(f" Download dir: {DOWNLOAD_DIR}") + print(f" Output file: {OUTPUT_FILE}") + print(f" Cleanup temp files: True (default)") + + output_path, df = extract_and_post_process( + work_frame=work_frame, + page=page1, + order_ids=test_order_ids, + download_dir=str(DOWNLOAD_DIR), + output_file=str(OUTPUT_FILE), + batch_size=100, + verbose=True, + cleanup_temp_files=True, # Default: True, set to False to keep temp files + ) + + print("\n[6/6] Verifying results...") + print(f"[OK] Extraction completed successfully") + print(f" Output file: {output_path}") + print(f" Total records: {len(df)}") + print(f" Columns: {list(df.columns)}") + + # Verify output file exists + if not Path(output_path).exists(): + raise FileNotFoundError(f"Output file not found: {output_path}") + print(f"[OK] Output file verified: {output_path}") + + # Show sample data + print("\n" + "=" * 60) + print("Sample Data (first 3 rows, first 5 columns):") + print("=" * 60) + # Use unicode encoding for display + sample = df.head(3).iloc[:, :5] + print(sample.to_string()) + print("\n[OK] Data extraction verified - Chinese characters displayed correctly in console encoding") + + finally: + # Cleanup browser + print("\n\n[Cleanup] Closing browser...") + context.close() + browser.close() + print("[OK] Browser closed") + + print("\n" + "=" * 60) + print("[SUCCESS] Integration test completed successfully!") + print("=" * 60) + print(f"\nOutputs:") + print(f" Raw Excel files: {DOWNLOAD_DIR}/*.xlsx") + print(f" Merged result: {OUTPUT_FILE}") + print(f" Total records: {len(df)}") + print("=" * 60) + +except Exception as e: + print(f"\n[ERROR] Integration test failed!") + print(f"Error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..6423ece --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1,11 @@ +""" +Utils package for BIPAuto automation framework. +""" + +from .auth import login, logout + +__all__ = [ + # Auth module + "login", + "logout", +] diff --git a/utils/discrete_material_plan/excel_converter.py b/utils/discrete_material_plan/excel_converter.py new file mode 100644 index 0000000..43f5ea3 --- /dev/null +++ b/utils/discrete_material_plan/excel_converter.py @@ -0,0 +1,280 @@ +""" +Excel 报表数据转换工具组件 +将 Excel 报表数据转换为数据库记录形式 +""" + +import pandas as pd +import openpyxl +from typing import List, Dict, Optional +import os + + +class ExcelConverter: + """Excel 报表数据转换器""" + + # 字段名称映射（解决字段名冲突） + FIELD_NAME_MAPPING = {"计划数量": "产品计划数量", "单位": "产品单位"} + + def __init__(self, verbose: bool = True): + """ + 初始化转换器 + + Args: + verbose: 是否打印详细日志 + """ + self.verbose = verbose + + def _print(self, *args, **kwargs): + """打印日志（如果 verbose=True）""" + if self.verbose: + print(*args, **kwargs) + + def convert(self, input_file: str, output_file: str = None) -> pd.DataFrame: + """ + 转换 Excel 文件 + + Args: + input_file: 输入文件路径 + output_file: 输出文件路径（可选，不指定则不保存） + + Returns: + 转换后的 DataFrame + """ + # 处理输出文件名 + if output_file: + output_file = self._handle_output_file(output_file) + + # 读取工作表 + wb = openpyxl.load_workbook(input_file) + ws = wb.active + + # 解析订单数据 + orders = self._parse_sheet(ws) + + # 转换为 DataFrame + df = self._convert_to_dataframe(orders) + + if not df.empty: + # 保存文件 + if output_file: + df.to_excel(output_file, index=False) + + # 打印汇总报告 + self._print("=" * 60) + self._print("转换完成") + self._print("=" * 60) + self._print(f"订单数: {len(orders)}") + self._print(f"数据行数: {len(df)}") + self._print(f"输出文件: {output_file if output_file else 'N/A'}") + self._print("=" * 60) + + return df + + def _handle_output_file(self, output_file: str) -> str: + """ + 处理输出文件，如果文件存在则尝试删除 + + Args: + output_file: 输出文件路径 + + Returns: + 实际使用的输出文件路径 + """ + if os.path.exists(output_file): + try: + os.remove(output_file) + except PermissionError: + self._print(f"警告: 无法删除 {output_file}，可能文件被其他程序打开") + # 修改文件名 + base, ext = os.path.splitext(output_file) + output_file = f"{base}_new{ext}" + return output_file + + def _parse_sheet(self, ws) -> List[Dict]: + """ + 解析一个工作表，返回所有订单的数据 + + 每个订单包含: + - order_info: 订单头信息（包括页脚） + - materials: 物料数据列表 + + Args: + ws: openpyxl 工作表对象 + + Returns: + 订单列表 + """ + orders = [] + all_rows = list(ws.iter_rows(values_only=True)) + + # 逐行扫描，按订单结构解析 + i = 0 + while i < len(all_rows): + row = all_rows[i] + + # 检查是否是订单标题行 + if row and "离散备料计划" in str(row[0]): + # 解析订单头信息（接下来的4行） + order_info = {} + for j in range(1, 5): + if i + j < len(all_rows) and all_rows[i + j]: + self._parse_header_row(all_rows[i + j], order_info) + + # 跳过空行，找到表格标题行 + table_row = i + 5 + while table_row < len(all_rows) and ( + not all_rows[table_row] or not all_rows[table_row][0] + ): + table_row += 1 + + # 检查是否是表格标题行 + if ( + table_row < len(all_rows) + and all_rows[table_row] + and all_rows[table_row][0] == "序号" + ): + # 检查表头下一行是否为空，判断是否存在数据 + next_row = table_row + 1 + is_empty_row = ( + next_row < len(all_rows) + and all_rows[next_row] + and all( + cell is None or str(cell).strip() == "" + for cell in all_rows[next_row] + ) + ) + + if is_empty_row: + # 没有数据，查找页脚信息 + materials = [] + footer_info = {} + data_row = next_row + 1 + while data_row < len(all_rows) and all_rows[data_row]: + if all_rows[data_row][0] and ( + "制单人" in str(all_rows[data_row][0]) + or "打印人" in str(all_rows[data_row][0]) + ): + self._parse_header_row(all_rows[data_row], footer_info) + if ( + data_row + 1 < len(all_rows) + and all_rows[data_row + 1] + ): + self._parse_header_row( + all_rows[data_row + 1], footer_info + ) + break + data_row += 1 + + orders.append( + { + "order_info": {**order_info, **footer_info}, + "materials": materials, + } + ) + else: + # 有数据，开始提取物料 + materials = [] + footer_info = {} # 页脚信息 + data_row = table_row + 1 + while data_row < len(all_rows) and all_rows[data_row]: + # 检查是否是页脚信息（制单人、打印人） + if all_rows[data_row + 1][0] and "制单人" in str( + all_rows[data_row + 1][0] + ): + # 解析页脚信息 + self._parse_header_row(all_rows[data_row], footer_info) + # 检查下一行是否也是页脚信息 + if ( + data_row + 1 < len(all_rows) + and all_rows[data_row + 1] + ): + self._parse_header_row( + all_rows[data_row + 1], footer_info + ) + break + + # 提取物料数据 + material_row = all_rows[data_row] + material = { + "序号": material_row[0], + "材料编码": material_row[1], + "材料名称": material_row[2], + "规格": material_row[3], + "型号": material_row[4], + "图号": material_row[5], + "物料材质": material_row[6], + "计划数量": material_row[7], + "单位": material_row[8], + "需用日期": material_row[9], + "发料仓库": material_row[10], + "单位用量": material_row[11], + "累计出库数量": material_row[12], + } + materials.append(material) + + data_row += 1 + + orders.append( + { + "order_info": {**order_info, **footer_info}, + "materials": materials, + } + ) + + i += 1 + + return orders + + def _parse_header_row(self, row: tuple, info: Dict): + """ + 解析订单头信息的一行（字段名和值交错排列） + + Args: + row: 行数据 + info: 存储解析结果的字典 + """ + i = 0 + while i < len(row): + cell = row[i] + if cell and str(cell).strip() and "：" in str(cell): + # 找到字段名 + field_name = str(cell).replace("：", "").strip() + + # 应用字段名映射 + if field_name in self.FIELD_NAME_MAPPING: + field_name = self.FIELD_NAME_MAPPING[field_name] + + # 跳过空单元格，找到第一个非字段名的值 + j = i + 1 + while j < len(row) and ( + not row[j] or not str(row[j]).strip() or "：" in str(row[j]) + ): + j += 1 + if j < len(row) and row[j] and not "：" in str(row[j]): + info[field_name] = str(row[j]).strip() + # 跳过已处理的值，继续找下一个字段名 + i = j + 1 + else: + i += 1 + + def _convert_to_dataframe(self, orders: List[Dict]) -> pd.DataFrame: + """ + 将订单数据转换为扁平化的 DataFrame + + Args: + orders: 订单列表 + + Returns: + 扁平化的 DataFrame + """ + all_records = [] + + for order in orders: + order_info = order["order_info"] + materials = order["materials"] + + for material in materials: + record = {**order_info, **material} + all_records.append(record) + + return pd.DataFrame(all_records) diff --git a/utils/discrete_material_plan/extractor.py b/utils/discrete_material_plan/extractor.py index 5172546..df3eaa4 100644 --- a/utils/discrete_material_plan/extractor.py +++ b/utils/discrete_material_plan/extractor.py @@ -144,6 +144,7 @@ def post_process_downloads( downloaded_files: List[str], output_file: str, verbose: bool = True, + cleanup_temp_files: bool = True, ) -> Tuple[str, pd.DataFrame]: """ Convert and merge downloaded Excel files into structured DataFrame. @@ -154,6 +155,7 @@ def post_process_downloads( downloaded_files: List of paths to downloaded Excel files output_file: Path to save merged Excel result verbose: Whether to print progress messages + cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True) Returns: Tuple of (output_file_path, merged_dataframe) @@ -162,7 +164,8 @@ def post_process_downloads( >>> output_path, df = post_process_downloads( ... downloaded_files=["batch_1.xlsx", "batch_2.xlsx"], ... output_file="merged.xlsx", - ... verbose=True + ... verbose=True, + ... cleanup_temp_files=True ... ) """ from .excel_converter import ExcelConverter @@ -194,9 +197,37 @@ def post_process_downloads( print(f"Merged result saved to: {output_path}") print(f"Total rows: {len(merged_df)}") + # Cleanup temporary downloaded files + if cleanup_temp_files: + _cleanup_temp_files(downloaded_files, verbose) + return str(output_path), merged_df +def _cleanup_temp_files(downloaded_files: List[str], verbose: bool = True) -> int: + """ + Remove temporary downloaded files. + + Args: + downloaded_files: List of file paths to delete + verbose: Whether to print progress messages + + Returns: + Number of files successfully deleted + """ + deleted_count = 0 + for file_path in downloaded_files: + try: + Path(file_path).unlink() + deleted_count += 1 + if verbose: + print(f"Deleted temp file: {file_path}") + except Exception as e: + if verbose: + print(f"Warning: Could not delete {file_path}: {e}") + return deleted_count + + def extract_and_post_process( work_frame: Frame, page: Page, @@ -205,6 +236,7 @@ def extract_and_post_process( output_file: str, batch_size: int = 10, verbose: bool = True, + cleanup_temp_files: bool = True, ) -> Tuple[str, pd.DataFrame]: """ Complete extraction workflow: download batches + post-process to merged Excel. @@ -220,6 +252,7 @@ def extract_and_post_process( output_file: Path for final merged Excel output batch_size: Maximum order IDs per batch verbose: Whether to print progress messages + cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True) Returns: Tuple of (output_file_path, merged_dataframe) @@ -229,7 +262,8 @@ def extract_and_post_process( >>> browser, context, page, main_frame = login(...) >>> work_frame, page1 = navigate_to_discrete_material_page(main_frame, page) >>> output_path, df = extract_and_post_process( - ... work_frame, page1, order_ids, "/downloads", "output.xlsx" + ... work_frame, page1, order_ids, "/downloads", "output.xlsx", + ... cleanup_temp_files=True ... ) >>> context.close() >>> browser.close() @@ -254,6 +288,7 @@ def extract_and_post_process( downloaded_files=downloaded_files, output_file=output_file, verbose=verbose, + cleanup_temp_files=cleanup_temp_files, ) return output_path, merged_df @@ -292,6 +327,7 @@ def extract_from_file( output_file: str, batch_size: int = 10, verbose: bool = True, + cleanup_temp_files: bool = True, ) -> Tuple[str, pd.DataFrame]: """ Extract data from order IDs in a file and post-process to merged Excel. @@ -306,6 +342,7 @@ def extract_from_file( output_file: Path for final merged Excel output batch_size: Maximum order IDs per batch verbose: Whether to print progress messages + cleanup_temp_files: Whether to delete temporary downloaded files after processing (default: True) Returns: Tuple of (output_file_path, merged_dataframe) @@ -315,7 +352,8 @@ def extract_from_file( >>> order_ids = read_order_ids_from_file("orders.txt") >>> # Extract and process >>> output_path, df = extract_from_file( - ... "orders.txt", work_frame, page, "/downloads", "output.xlsx" + ... "orders.txt", work_frame, page, "/downloads", "output.xlsx", + ... cleanup_temp_files=True ... ) """ order_ids = read_order_ids_from_file(id_file) @@ -331,4 +369,5 @@ def extract_from_file( output_file=output_file, batch_size=batch_size, verbose=verbose, + cleanup_temp_files=cleanup_temp_files, ) diff --git a/utils/discrete_material_plan/extractor_core.py b/utils/discrete_material_plan/extractor_core.py new file mode 100644 index 0000000..af5ea3b --- /dev/null +++ b/utils/discrete_material_plan/extractor_core.py @@ -0,0 +1,164 @@ +""" +Core Web Operations Module +Pure functions for Yonyou BIP discrete material plan maintenance page interactions. +All functions are stateless and accept required parameters explicitly. +""" + +import re +import os +from typing import List +from playwright.sync_api import Page, Frame, TimeoutError + + +def navigate_to_discrete_material_page(main_frame: Frame, page: Page) -> tuple[Frame, Page]: + """ + Navigate to the discrete material plan maintenance page. + + Args: + main_frame: The main forwardFrame iframe + page: The Playwright page object + + Returns: + tuple: (work_frame, page1) - The work iframe and the new popup page + """ + # Click icon to open menu + main_frame.locator("i").first.click() + + # Wait for popup and click menu item + with page.expect_popup() as page1_info: + main_frame.get_by_title("离散备料计划维护", exact=True).first.click() + + page1 = page1_info.value + + # Get nested iframes + f_frame = page1.locator("#forwardFrame").content_frame + inner_frame_locator = f_frame.locator("#mainiframe") + inner_frame_locator.wait_for(state="visible", timeout=15000) + work_frame = inner_frame_locator.content_frame + + return work_frame, page1 + + +def setup_query_interface(work_frame: Frame) -> None: + """ + Initialize the query interface by selecting order number query tab. + + Args: + work_frame: The inner work iframe containing the query interface + """ + # Open search panel + work_frame.locator(".search-name-wrapper > .iconfont").click() + + # Select order number query + work_frame.get_by_text("订单号查询").click() + + # Select "All" tab + work_frame.get_by_role("tab", name="全部").click() + + # Set page size to 5000 + input_box = work_frame.locator("#rc_select_0") + input_box.fill("5000") + input_box.press("Enter") + + +def fill_and_search_orders(work_frame: Frame, order_ids: List[str]) -> None: + """ + Fill order IDs into the search textbox and trigger search. + + Args: + work_frame: The work iframe containing the search form + order_ids: List of order IDs to search for + """ + textbox = work_frame.get_by_role("textbox", name="来源生产订单号") + + # Clear and fill order IDs + textbox.fill("") + textbox.fill(",".join(order_ids)) + + # Click search button + work_frame.locator(".search-component-searchBtn").click() + + # Wait for loading to complete + loading_locator = work_frame.locator("div").filter(has_text="加载中").nth(1) + try: + loading_locator.wait_for(state="visible", timeout=3000) + loading_locator.wait_for(state="hidden", timeout=0) + except TimeoutError: + pass + + +def download_batch_data( + work_frame: Frame, + page: Page, + order_ids: List[str], + batch_index: int, + download_dir: str +) -> str: + """ + Execute the download workflow for a single batch of order IDs. + + Args: + work_frame: The work iframe containing the data grid + page: The Playwright page object for download handling + order_ids: List of order IDs to download + batch_index: Zero-based batch index for naming the output file + download_dir: Directory path to save the downloaded file + + Returns: + str: Full path to the downloaded file + + Raises: + TimeoutError: If UI elements are not found or operations timeout + """ + # Step 1: Select first row + work_frame.get_by_role("row", name="序号").get_by_label("").click() + + # Step 2: Hover over "More" button + work_frame.get_by_role("button", name="更多").hover() + + # Step 3: Click "Export" + work_frame.get_by_text("输出", exact=True).click() + + # Step 4: Set row threshold + threshold_box = ( + work_frame.locator("div") + .filter(has_text=re.compile(r"^行数阈值$")) + .locator("input[type='text']") + ) + threshold_box.fill("300000") + + # Step 5: Trigger download and save file + download_filename = f"temp_batch_{batch_index + 1}.xlsx" + download_path = os.path.join(download_dir, download_filename) + + with page.expect_download() as download_info: + work_frame.get_by_role("button", name="确定(Y)").click() + + download = download_info.value + download.save_as(download_path) + + return download_path + + +def execute_batch_download_workflow( + work_frame: Frame, + page: Page, + order_ids: List[str], + batch_index: int, + download_dir: str +) -> str: + """ + Complete workflow: fill orders, search, and download for a single batch. + + Args: + work_frame: The work iframe containing the search form and data grid + page: The Playwright page object for download handling + order_ids: List of order IDs for this batch + batch_index: Zero-based batch index for naming the output file + download_dir: Directory path to save the downloaded file + + Returns: + str: Full path to the downloaded file + """ + fill_and_search_orders(work_frame, order_ids) + return download_batch_data(work_frame, page, order_ids, batch_index, download_dir)