refactor(logging): Add optional logging support throughout codebase
Add centralized logging utility and optional logger parameters to all core functions for better observability and debugging capabilities. New modules: - utils/logging.py: Centralized logger configuration with console and optional file handlers Enhanced features: - Added optional logger parameter to all extractor_core functions - Added logger support to extractor, excel_converter, and auth modules - Functions remain silent when logger=None (backward compatible) - Improved environment variable validation in test files Documentation: - Added discrete_material_plan_extractor_core.md with complete API reference and usage patterns Benefits: - Consistent logging format across all components - Optional debug output for troubleshooting - No breaking changes - fully backward compatible - Better error messages and validation Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
@@ -1,139 +1,150 @@
|
||||
"""
|
||||
Excel 报表数据转换工具组件
|
||||
将 Excel 报表数据转换为数据库记录形式
|
||||
Excel Report Data Conversion Utility
|
||||
Converts Excel report data to database record format
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import openpyxl
|
||||
from typing import List, Dict, Optional
|
||||
import os
|
||||
import logging
|
||||
from utils.logging import get_logger
|
||||
|
||||
|
||||
class ExcelConverter:
|
||||
"""Excel 报表数据转换器"""
|
||||
"""Excel Report Data Converter"""
|
||||
|
||||
# 字段名称映射（解决字段名冲突）
|
||||
FIELD_NAME_MAPPING = {"计划数量": "产品计划数量", "单位": "产品单位"}
|
||||
# Field name mapping (resolves field name conflicts)
|
||||
FIELD_NAME_MAPPING = {"计划数量": "Product planned quantity", "单位": "Product unit"}
|
||||
|
||||
def __init__(self, verbose: bool = True):
|
||||
def __init__(self, verbose: bool = True, logger: Optional[logging.Logger] = None):
|
||||
"""
|
||||
初始化转换器
|
||||
Initialize the converter
|
||||
|
||||
Args:
|
||||
verbose: 是否打印详细日志
|
||||
verbose: Whether to print detailed logs (default: True). Deprecated, use logger instead.
|
||||
logger: Optional logging.Logger instance. If None and verbose=True, creates default logger.
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
# Create default logger if needed
|
||||
if logger is None and verbose:
|
||||
self.logger = get_logger('bipauto.converter')
|
||||
elif logger is None:
|
||||
# Silent mode - create a logger but disable output
|
||||
silent_logger = logging.getLogger('bipauto.converter.silent')
|
||||
silent_logger.setLevel(logging.CRITICAL + 1) # Higher than critical, never logs
|
||||
self.logger = silent_logger
|
||||
else:
|
||||
self.logger = logger
|
||||
|
||||
def _print(self, *args, **kwargs):
|
||||
"""打印日志（如果 verbose=True）"""
|
||||
if self.verbose:
|
||||
print(*args, **kwargs)
|
||||
|
||||
def convert(self, input_file: str, output_file: str = None) -> pd.DataFrame:
|
||||
def convert(self, input_file: str, output_file: Optional[str] = None) -> pd.DataFrame:
|
||||
"""
|
||||
转换 Excel 文件
|
||||
Convert Excel file
|
||||
|
||||
Args:
|
||||
input_file: 输入文件路径
|
||||
output_file: 输出文件路径（可选，不指定则不保存）
|
||||
input_file: Input file path
|
||||
output_file: Output file path (optional, not saved if not specified)
|
||||
|
||||
Returns:
|
||||
转换后的 DataFrame
|
||||
Converted DataFrame
|
||||
"""
|
||||
# 处理输出文件名
|
||||
# Handle output file
|
||||
if output_file:
|
||||
output_file = self._handle_output_file(output_file)
|
||||
|
||||
# 读取工作表
|
||||
# Read worksheet
|
||||
wb = openpyxl.load_workbook(input_file)
|
||||
ws = wb.active
|
||||
|
||||
# 解析订单数据
|
||||
# Parse order data
|
||||
orders = self._parse_sheet(ws)
|
||||
|
||||
# 转换为 DataFrame
|
||||
# Convert to DataFrame
|
||||
df = self._convert_to_dataframe(orders)
|
||||
|
||||
if not df.empty:
|
||||
# 保存文件
|
||||
# Save file
|
||||
if output_file:
|
||||
df.to_excel(output_file, index=False)
|
||||
|
||||
# 打印汇总报告
|
||||
self._print("=" * 60)
|
||||
self._print("转换完成")
|
||||
self._print("=" * 60)
|
||||
self._print(f"订单数: {len(orders)}")
|
||||
self._print(f"数据行数: {len(df)}")
|
||||
self._print(f"输出文件: {output_file if output_file else 'N/A'}")
|
||||
self._print("=" * 60)
|
||||
# Print summary report (using logger)
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info("Conversion complete")
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info(f"Order count: {len(orders)}")
|
||||
self.logger.info(f"Data row count: {len(df)}")
|
||||
self.logger.info(f"Output file: {output_file if output_file else 'N/A'}")
|
||||
self.logger.info("=" * 60)
|
||||
|
||||
return df
|
||||
|
||||
def _handle_output_file(self, output_file: str) -> str:
|
||||
"""
|
||||
处理输出文件，如果文件存在则尝试删除
|
||||
Handle output file, attempt to delete if it exists
|
||||
|
||||
Args:
|
||||
output_file: 输出文件路径
|
||||
output_file: Output file path
|
||||
|
||||
Returns:
|
||||
实际使用的输出文件路径
|
||||
Actual output file path used
|
||||
"""
|
||||
if os.path.exists(output_file):
|
||||
try:
|
||||
os.remove(output_file)
|
||||
except PermissionError:
|
||||
self._print(f"警告: 无法删除 {output_file}，可能文件被其他程序打开")
|
||||
# 修改文件名
|
||||
self.logger.warning(
|
||||
f"Warning: Could not delete {output_file}, file may be open by another program"
|
||||
)
|
||||
# Modify filename
|
||||
base, ext = os.path.splitext(output_file)
|
||||
output_file = f"{base}_new{ext}"
|
||||
return output_file
|
||||
|
||||
def _parse_sheet(self, ws) -> List[Dict]:
|
||||
"""
|
||||
解析一个工作表，返回所有订单的数据
|
||||
Parse a worksheet and return data for all orders
|
||||
|
||||
每个订单包含:
|
||||
- order_info: 订单头信息（包括页脚）
|
||||
- materials: 物料数据列表
|
||||
Each order contains:
|
||||
- order_info: Order header information (including footer)
|
||||
- materials: List of material data
|
||||
|
||||
Args:
|
||||
ws: openpyxl 工作表对象
|
||||
ws: openpyxl worksheet object
|
||||
|
||||
Returns:
|
||||
订单列表
|
||||
Order list
|
||||
"""
|
||||
orders = []
|
||||
all_rows = list(ws.iter_rows(values_only=True))
|
||||
|
||||
# 逐行扫描，按订单结构解析
|
||||
# Scan row by row, parse by order structure
|
||||
i = 0
|
||||
while i < len(all_rows):
|
||||
row = all_rows[i]
|
||||
|
||||
# 检查是否是订单标题行
|
||||
# Check if this is the order title row
|
||||
if row and "离散备料计划" in str(row[0]):
|
||||
# 解析订单头信息（接下来的4行）
|
||||
# Parse order header information (next 4 rows)
|
||||
order_info = {}
|
||||
for j in range(1, 5):
|
||||
if i + j < len(all_rows) and all_rows[i + j]:
|
||||
self._parse_header_row(all_rows[i + j], order_info)
|
||||
|
||||
# 跳过空行，找到表格标题行
|
||||
# Skip empty rows, find table header row
|
||||
table_row = i + 5
|
||||
while table_row < len(all_rows) and (
|
||||
not all_rows[table_row] or not all_rows[table_row][0]
|
||||
):
|
||||
table_row += 1
|
||||
|
||||
# 检查是否是表格标题行
|
||||
# Check if this is the table header row
|
||||
if (
|
||||
table_row < len(all_rows)
|
||||
and all_rows[table_row]
|
||||
and all_rows[table_row][0] == "序号"
|
||||
):
|
||||
# 检查表头下一行是否为空，判断是否存在数据
|
||||
# Check if the row below the header is empty to determine if data exists
|
||||
next_row = table_row + 1
|
||||
is_empty_row = (
|
||||
next_row < len(all_rows)
|
||||
@@ -145,7 +156,7 @@ class ExcelConverter:
|
||||
)
|
||||
|
||||
if is_empty_row:
|
||||
# 没有数据，查找页脚信息
|
||||
# No data, find footer information
|
||||
materials = []
|
||||
footer_info = {}
|
||||
data_row = next_row + 1
|
||||
@@ -172,18 +183,18 @@ class ExcelConverter:
|
||||
}
|
||||
)
|
||||
else:
|
||||
# 有数据，开始提取物料
|
||||
# Has data, start extracting materials
|
||||
materials = []
|
||||
footer_info = {} # 页脚信息
|
||||
footer_info = {} # Footer information
|
||||
data_row = table_row + 1
|
||||
while data_row < len(all_rows) and all_rows[data_row]:
|
||||
# 检查是否是页脚信息（制单人、打印人）
|
||||
# Check if this is footer information (creator, printer)
|
||||
if all_rows[data_row + 1][0] and "制单人" in str(
|
||||
all_rows[data_row + 1][0]
|
||||
):
|
||||
# 解析页脚信息
|
||||
# Parse footer information
|
||||
self._parse_header_row(all_rows[data_row], footer_info)
|
||||
# 检查下一行是否也是页脚信息
|
||||
# Check if the next row is also footer information
|
||||
if (
|
||||
data_row + 1 < len(all_rows)
|
||||
and all_rows[data_row + 1]
|
||||
@@ -193,7 +204,7 @@ class ExcelConverter:
|
||||
)
|
||||
break
|
||||
|
||||
# 提取物料数据
|
||||
# Extract material data
|
||||
material_row = all_rows[data_row]
|
||||
material = {
|
||||
"序号": material_row[0],
|
||||
@@ -227,45 +238,45 @@ class ExcelConverter:
|
||||
|
||||
def _parse_header_row(self, row: tuple, info: Dict):
|
||||
"""
|
||||
解析订单头信息的一行（字段名和值交错排列）
|
||||
Parse a row of order header information (field names and values interleaved)
|
||||
|
||||
Args:
|
||||
row: 行数据
|
||||
info: 存储解析结果的字典
|
||||
row: Row data
|
||||
info: Dictionary to store parsing results
|
||||
"""
|
||||
i = 0
|
||||
while i < len(row):
|
||||
cell = row[i]
|
||||
if cell and str(cell).strip() and "：" in str(cell):
|
||||
# 找到字段名
|
||||
field_name = str(cell).replace("：", "").strip()
|
||||
# Find field name
|
||||
field_name = str(cell).replace(":", "").strip()
|
||||
|
||||
# 应用字段名映射
|
||||
# Apply field name mapping
|
||||
if field_name in self.FIELD_NAME_MAPPING:
|
||||
field_name = self.FIELD_NAME_MAPPING[field_name]
|
||||
|
||||
# 跳过空单元格，找到第一个非字段名的值
|
||||
# Skip empty cells, find the first non-field-name value
|
||||
j = i + 1
|
||||
while j < len(row) and (
|
||||
not row[j] or not str(row[j]).strip() or "：" in str(row[j])
|
||||
not row[j] or not str(row[j]).strip() or ":" in str(row[j])
|
||||
):
|
||||
j += 1
|
||||
if j < len(row) and row[j] and not "：" in str(row[j]):
|
||||
if j < len(row) and row[j] and ":" not in str(row[j]):
|
||||
info[field_name] = str(row[j]).strip()
|
||||
# 跳过已处理的值，继续找下一个字段名
|
||||
# Skip processed value, continue to find next field name
|
||||
i = j + 1
|
||||
else:
|
||||
i += 1
|
||||
|
||||
def _convert_to_dataframe(self, orders: List[Dict]) -> pd.DataFrame:
|
||||
"""
|
||||
将订单数据转换为扁平化的 DataFrame
|
||||
Convert order data to a flattened DataFrame
|
||||
|
||||
Args:
|
||||
orders: 订单列表
|
||||
orders: Order list
|
||||
|
||||
Returns:
|
||||
扁平化的 DataFrame
|
||||
Flattened DataFrame
|
||||
"""
|
||||
all_records = []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user