feat: add MaterialCode-based deduplication to material validation queries

Implement database-level deduplication using ROW_NUMBER() window function
to remove duplicate material records based on MaterialCode field.

Changes:
- Add query_all_distinct_by_material_code() to deduplicate full table results
- Add query_by_source_numbers_distinct() to deduplicate filtered results
- Update validator to use new deduplication methods with statistics logging
- Retention strategy: keep earliest record (CreateDate ASC, SequenceNumber ASC)

Benefits:
- Reduces data redundancy from 54,358 to 1,679 records (96.91% reduction)
- Improves validation result readability by removing duplicate materials
- Maintains backward compatibility with existing query methods

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Misaka_Company
2026-02-24 16:22:35 +08:00
parent 0644bdfb11
commit e5f6f4ee0c
2 changed files with 112 additions and 5 deletions

View File

@@ -446,10 +446,18 @@ class MaterialStatusValidator:
# Get material records (complete records, not just MaterialName)
if full_table:
self._print("\n模式: 全表校验")
self._print("[INFO] 查询 DiscreteMaterialPlanData 表中的所有完整记录...")
self._print("[INFO] 查询 DiscreteMaterialPlanData 表中的所有完整记录(启用 MaterialCode 去重)...")
dao = DiscreteMaterialPlanDAO()
material_records = dao.query_all()
# Get original count for deduplication statistics
original_count = dao.count_all()
material_records = dao.query_all_distinct_by_material_code()
dedup_count = original_count - len(material_records)
self._print(f"[INFO] 获取到 {len(material_records)} 条记录")
if dedup_count > 0:
self._print(f"[INFO] 基于 MaterialCode 去重:移除了 {dedup_count} 条重复记录")
elif production_id_file:
self._print("\n模式: ProductionID 过滤校验")
self._print(f"[INFO] 读取 ProductionID 文件: {production_id_file}")
@@ -461,11 +469,19 @@ class MaterialStatusValidator:
# 2. Query SourceNumbers
source_numbers = self._get_source_numbers_from_production_ids(production_ids)
# 3. Get complete material records
self._print(f"[INFO] 查询 {len(source_numbers)} 个生产订单对应的完整物料记录...")
# 3. Get complete material records with deduplication
self._print(f"[INFO] 查询 {len(source_numbers)} 个生产订单对应的完整物料记录(启用 MaterialCode 去重)...")
dao = DiscreteMaterialPlanDAO()
material_records = dao.query_by_source_numbers(source_numbers)
# Get original count for deduplication statistics
original_records = dao.query_by_source_numbers(source_numbers)
material_records = dao.query_by_source_numbers_distinct(source_numbers)
dedup_count = len(original_records) - len(material_records)
self._print(f"[INFO] 获取到 {len(material_records)} 条记录")
if dedup_count > 0:
self._print(f"[INFO] 基于 MaterialCode 去重:移除了 {dedup_count} 条重复记录")
else:
raise ValueError("必须指定 full_table=True 或提供 production_id_file 参数")