refactor(cleaner): batch query and controlled parallel order processing

This commit is contained in:
test
2026-03-16 21:47:13 +08:00
parent 9248be6310
commit 6a2fba0e57
8 changed files with 484 additions and 236 deletions

View File

@@ -210,7 +210,11 @@ export function registerCleanerHandlers(): void {
} }
} }
log.info('Starting cleaning', { orderCount: validOrderNumbers.length }) log.info('Starting cleaning', {
orderCount: validOrderNumbers.length,
queryBatchSize: input.queryBatchSize ?? 100,
processConcurrency: input.processConcurrency ?? 1
})
const result = await cleaner.clean(modifiedInput) const result = await cleaner.clean(modifiedInput)
if (warnings.length > 0) { if (warnings.length > 0) {
@@ -249,6 +253,8 @@ export function registerCleanerHandlers(): void {
metadata: { metadata: {
orderCount: validOrderNumbers.length, orderCount: validOrderNumbers.length,
dryRun: input.dryRun ?? false, dryRun: input.dryRun ?? false,
queryBatchSize: input.queryBatchSize ?? 100,
processConcurrency: input.processConcurrency ?? 1,
materialsDeleted: result.materialsDeleted, materialsDeleted: result.materialsDeleted,
materialsSkipped: result.materialsSkipped, materialsSkipped: result.materialsSkipped,
errorCount: result.errors.length errorCount: result.errors.length

View File

@@ -12,7 +12,9 @@ export const CleanerInputSchema = z.object({
.array(z.string().min(1, 'Order number cannot be empty')) .array(z.string().min(1, 'Order number cannot be empty'))
.min(1, 'At least one order number is required'), .min(1, 'At least one order number is required'),
materialCodes: z.array(z.string().min(1, 'Material code cannot be empty')), materialCodes: z.array(z.string().min(1, 'Material code cannot be empty')),
dryRun: z.boolean() dryRun: z.boolean(),
queryBatchSize: z.number().int().min(1).max(100).optional().default(100),
processConcurrency: z.number().int().min(1).max(20).optional().default(1)
// Note: onProgress is a function, not validated via Zod // Note: onProgress is a function, not validated via Zod
}) })

View File

@@ -7,12 +7,43 @@ import { createLogger } from '../logger'
const log = createLogger('CleanerService') const log = createLogger('CleanerService')
const DEFAULT_QUERY_BATCH_SIZE = 100
const MAX_QUERY_BATCH_SIZE = 100
const DEFAULT_PROCESS_CONCURRENCY = 1
const MAX_PROCESS_CONCURRENCY = 20
interface RetryResult { interface RetryResult {
retriedOrders: number retriedOrders: number
successfulRetries: number successfulRetries: number
updatedDetails: OrderCleanDetail[] updatedDetails: OrderCleanDetail[]
} }
interface ProgressState {
completedOrders: number
totalOrders: number
}
class AsyncMutex {
private queue: Promise<void> = Promise.resolve()
async runExclusive<T>(task: () => Promise<T>): Promise<T> {
let release!: () => void
const next = new Promise<void>((resolve) => {
release = resolve
})
const previous = this.queue
this.queue = this.queue.then(() => next)
await previous
try {
return await task()
} finally {
release()
}
}
}
/** /**
* Cleaner Service Options * Cleaner Service Options
*/ */
@@ -31,11 +62,53 @@ export interface ShouldDeleteParams {
deleteSet: Set<string> deleteSet: Set<string>
} }
function clampNumber(value: number | undefined, fallback: number, min: number, max: number): number {
if (!Number.isFinite(value)) {
return fallback
}
return Math.min(max, Math.max(min, Math.trunc(value ?? fallback)))
}
export function createBatches<T>(items: T[], batchSize: number): T[][] {
const batches: T[][] = []
for (let i = 0; i < items.length; i += batchSize) {
batches.push(items.slice(i, i + batchSize))
}
return batches
}
export function getMissingOrders(inputOrders: string[], processedOrders: Set<string>): string[] {
const uniqueInputOrders = Array.from(new Set(inputOrders))
return uniqueInputOrders.filter((order) => !processedOrders.has(order))
}
export async function runWithConcurrency<T, R>(
items: T[],
concurrency: number,
worker: (item: T, index: number) => Promise<R>
): Promise<R[]> {
const results = new Array<R>(items.length)
const limit = Math.max(1, Math.trunc(concurrency))
let cursor = 0
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
while (true) {
const current = cursor
cursor += 1
if (current >= items.length) {
return
}
results[current] = await worker(items[current], current)
}
})
await Promise.all(runners)
return results
}
/** /**
* ERP Cleaner Service * ERP Cleaner Service
* Deletes specified materials from production orders in ERP system * Deletes specified materials from production orders in ERP system
*
* Reference: playwrite/utils/discrete_material_plan_cleaner.py
*/ */
export class CleanerService { export class CleanerService {
private authService: ErpAuthService private authService: ErpAuthService
@@ -55,27 +128,18 @@ export class CleanerService {
/** /**
* Determine if a material should be deleted * Determine if a material should be deleted
* Reference: Python lines 284-406
*
* Deletion conditions:
* 1. Material code must be in the delete set
* 2. Row number must NOT be in range 7000-7999
* 3. Pending quantity must be empty
*/ */
shouldDeleteMaterial(params: ShouldDeleteParams): boolean { shouldDeleteMaterial(params: ShouldDeleteParams): boolean {
const { rowNumber, pendingQty, materialCode, deleteSet } = params const { rowNumber, pendingQty, materialCode, deleteSet } = params
// Check if material is in delete list
if (!deleteSet.has(materialCode)) { if (!deleteSet.has(materialCode)) {
return false return false
} }
// Check row number range (7000-7999 are protected)
if (rowNumber >= 7000 && rowNumber < 8000) { if (rowNumber >= 7000 && rowNumber < 8000) {
return false return false
} }
// Check pending quantity (must be empty)
if (pendingQty && pendingQty.trim() !== '') { if (pendingQty && pendingQty.trim() !== '') {
return false return false
} }
@@ -98,10 +162,6 @@ export class CleanerService {
return '未知原因' return '未知原因'
} }
/**
* Execute the cleaning process
* Reference: Python clean() method lines 456-525
*/
async clean(input: CleanerInput): Promise<CleanerResult> { async clean(input: CleanerInput): Promise<CleanerResult> {
const result: CleanerResult = { const result: CleanerResult = {
ordersProcessed: 0, ordersProcessed: 0,
@@ -115,90 +175,121 @@ export class CleanerService {
const totalOrders = input.orderNumbers.length const totalOrders = input.orderNumbers.length
const dryRun = input.dryRun ?? this.dryRun const dryRun = input.dryRun ?? this.dryRun
const queryBatchSize = clampNumber(
input.queryBatchSize,
DEFAULT_QUERY_BATCH_SIZE,
1,
MAX_QUERY_BATCH_SIZE
)
const processConcurrency = clampNumber(
input.processConcurrency,
DEFAULT_PROCESS_CONCURRENCY,
1,
MAX_PROCESS_CONCURRENCY
)
log.info('Starting cleaner', { log.info('Starting cleaner', {
totalOrders, totalOrders,
materialCount: input.materialCodes.length, materialCount: input.materialCodes.length,
dryRun dryRun,
queryBatchSize,
processConcurrency
}) })
// Create delete set for O(1) lookup
const deleteSet = new Set(input.materialCodes) const deleteSet = new Set(input.materialCodes)
let popupPage: Page | null = null
try { try {
const session = this.authService.getSession() const session = this.authService.getSession()
const navigation = await this.navigateToCleanerPage(session)
popupPage = navigation.popupPage
const { workFrame } = navigation
// Navigate to cleaner page
const { popupPage, workFrame } = await this.navigateToCleanerPage(session)
// Setup query interface
await this.setupQueryInterface(workFrame) await this.setupQueryInterface(workFrame)
// Process each order const orderBatches = createBatches(input.orderNumbers, queryBatchSize)
for (let i = 0; i < totalOrders; i++) { const popupMutex = new AsyncMutex()
const orderNumber = input.orderNumbers[i] const progressState: ProgressState = {
completedOrders: 0,
totalOrders
}
try { for (let batchIndex = 0; batchIndex < orderBatches.length; batchIndex++) {
log.debug('Processing order', { orderNumber, index: i + 1, total: totalOrders }) const batchOrders = orderBatches[batchIndex]
const detail = await this.processOrder({
workFrame, log.info('Processing cleaner batch', {
popupPage, batchIndex: batchIndex + 1,
orderNumber, totalBatches: orderBatches.length,
orderIndex: i, batchSize: batchOrders.length
totalOrders, })
deleteSet,
dryRun: input.dryRun ?? this.dryRun, await this.queryOrders(workFrame, batchOrders)
onProgress: input.onProgress await this.waitForLoading(workFrame)
const rows = workFrame.locator('tbody tr')
const rowCount = await rows.count()
const rowIndexes = Array.from({ length: rowCount }, (_, i) => i)
const processedOrderNumbersInBatch = new Set<string>()
await runWithConcurrency(rowIndexes, processConcurrency, async (rowIndex) => {
const openedDetailPage = await popupMutex.runExclusive(async () => {
return await this.openDetailPageFromRow(workFrame, popupPage!, rowIndex)
}) })
let detail: OrderCleanDetail
try {
detail = await this.processDetailPage({
detailPage: openedDetailPage,
deleteSet,
dryRun,
progressState,
onProgress: input.onProgress
})
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error'
detail = this.createErrorDetail(`BATCH_ROW_${rowIndex + 1}`, message)
} finally {
progressState.completedOrders += 1
}
result.details.push(detail) result.details.push(detail)
result.ordersProcessed++
if (detail.errors.length > 0) {
result.errors.push(`Order ${detail.orderNumber}: ${detail.errors.join('; ')}`)
return
}
result.ordersProcessed += 1
result.materialsDeleted += detail.materialsDeleted result.materialsDeleted += detail.materialsDeleted
result.materialsSkipped += detail.materialsSkipped result.materialsSkipped += detail.materialsSkipped
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error'
log.error('Order processing failed', { orderNumber, error: message })
result.errors.push(`Order ${orderNumber}: ${message}`)
// Add error detail if (this.isOrderNumber(detail.orderNumber)) {
result.details.push({ processedOrderNumbersInBatch.add(detail.orderNumber)
orderNumber, }
materialsDeleted: 0, })
materialsSkipped: 0,
errors: [message], const missingOrders = getMissingOrders(batchOrders, processedOrderNumbersInBatch)
skippedMaterials: [], for (const missingOrder of missingOrders) {
retryCount: 0, const missingMessage = '订单未出现在查询结果中'
retryAttempts: [], result.errors.push(`Order ${missingOrder}: ${missingMessage}`)
retriedAt: undefined, result.details.push(this.createErrorDetail(missingOrder, missingMessage))
retrySuccess: false
})
} }
} }
// Close popup page
await popupPage.close()
log.info('Cleaner completed', {
ordersProcessed: result.ordersProcessed,
materialsDeleted: result.materialsDeleted,
materialsSkipped: result.materialsSkipped,
errorCount: result.errors.length
})
// Retry failed orders
const retryResult = await this.retryFailedOrders({ const retryResult = await this.retryFailedOrders({
workFrame, workFrame,
popupPage, popupPage,
failedDetails: result.details.filter((d) => d.errors.length > 0), failedDetails: result.details.filter((d) => d.errors.length > 0 && this.isOrderNumber(d.orderNumber)),
deleteSet, deleteSet,
dryRun, dryRun,
onProgress: input.onProgress onProgress: input.onProgress
}) })
// Merge retry results
result.retriedOrders = retryResult.retriedOrders result.retriedOrders = retryResult.retriedOrders
result.successfulRetries = retryResult.successfulRetries result.successfulRetries = retryResult.successfulRetries
// Update details with retry information
retryResult.updatedDetails.forEach((updatedDetail) => { retryResult.updatedDetails.forEach((updatedDetail) => {
const index = result.details.findIndex((d) => d.orderNumber === updatedDetail.orderNumber) const index = result.details.findIndex((d) => d.orderNumber === updatedDetail.orderNumber)
if (index !== -1) { if (index !== -1) {
@@ -206,40 +297,47 @@ export class CleanerService {
} }
}) })
// Clear errors for successfully retried orders
const successfulRetryOrders = new Set( const successfulRetryOrders = new Set(
retryResult.updatedDetails.filter((d) => d.retrySuccess).map((d) => d.orderNumber) retryResult.updatedDetails.filter((d) => d.retrySuccess).map((d) => d.orderNumber)
) )
result.errors = result.errors.filter( result.errors = result.errors.filter(
(err) => !successfulRetryOrders.has(err.split(':')[0].replace('Order ', '')) (err) => !successfulRetryOrders.has(err.split(':')[0].replace('Order ', ''))
) )
log.info('Cleaner completed', {
ordersProcessed: result.ordersProcessed,
materialsDeleted: result.materialsDeleted,
materialsSkipped: result.materialsSkipped,
errorCount: result.errors.length
})
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error' const message = error instanceof Error ? error.message : 'Unknown error'
log.error('Cleaner failed', { error: message }) log.error('Cleaner failed', { error: message })
result.errors.push(`Clean failed: ${message}`) result.errors.push(`Clean failed: ${message}`)
} finally {
if (popupPage) {
try {
await popupPage.close()
} catch {
// Ignore close errors
}
}
} }
return result return result
} }
/**
* Navigate to the discrete production order maintenance page
* Reference: Python lines 476-486
*/
async navigateToCleanerPage( async navigateToCleanerPage(
session: ErpSession session: ErpSession
): Promise<{ popupPage: Page; workFrame: FrameLocator }> { ): Promise<{ popupPage: Page; workFrame: FrameLocator }> {
const { page, mainFrame } = session const { page, mainFrame } = session
// Click menu icon (Python line 476)
await mainFrame.locator('i').first().click() await mainFrame.locator('i').first().click()
// Click discrete production order menu item and expect popup (Python lines 477-479)
const popupPromise = page.waitForEvent('popup') const popupPromise = page.waitForEvent('popup')
await mainFrame.getByTitle('离散生产订单维护', { exact: true }).first().click() await mainFrame.getByTitle('离散生产订单维护', { exact: true }).first().click()
const popupPage = await popupPromise const popupPage = await popupPromise
// Get nested frame structure (Python lines 482-486)
const forwardFrameLocator = popupPage.locator('#forwardFrame') const forwardFrameLocator = popupPage.locator('#forwardFrame')
const fFrame = forwardFrameLocator.contentFrame() const fFrame = forwardFrameLocator.contentFrame()
@@ -247,91 +345,99 @@ export class CleanerService {
await innerFrameLocator.waitFor({ state: 'visible', timeout: 30000 }) await innerFrameLocator.waitFor({ state: 'visible', timeout: 30000 })
const workFrame = innerFrameLocator.contentFrame() const workFrame = innerFrameLocator.contentFrame()
// Wait for hot-key-head_list to be visible (Python line 484-486)
await workFrame.locator('#hot-key-head_list').waitFor({ state: 'visible', timeout: 30000 }) await workFrame.locator('#hot-key-head_list').waitFor({ state: 'visible', timeout: 30000 })
return { popupPage, workFrame } return { popupPage, workFrame }
} }
/**
* Setup query interface
* Reference: Python setup_query_interface() lines 445-454
*/
private async setupQueryInterface(innerFrame: FrameLocator): Promise<void> { private async setupQueryInterface(innerFrame: FrameLocator): Promise<void> {
// Click search icon (Python line 447)
await innerFrame.locator('.search-name-wrapper > .iconfont').click() await innerFrame.locator('.search-name-wrapper > .iconfont').click()
// Click "订单号查询" menu item (Python line 448)
await innerFrame.getByText('订单号查询').click() await innerFrame.getByText('订单号查询').click()
// Click "全部" tab (Python line 449)
await innerFrame.getByRole('tab', { name: '全部' }).click() await innerFrame.getByRole('tab', { name: '全部' }).click()
// Set limit to 5000 (Python lines 451-454)
const inputEl = innerFrame.locator('#rc_select_0') const inputEl = innerFrame.locator('#rc_select_0')
await inputEl.fill('5000') await inputEl.fill('5000')
await inputEl.press('Enter') await inputEl.press('Enter')
} }
/** private async queryOrders(workFrame: FrameLocator, orderNumbers: string[]): Promise<void> {
* Process a single order const textbox = workFrame.getByRole('textbox', { name: '生产订单号' })
* Reference: Python process_order() lines 171-443 await textbox.fill(orderNumbers.join(','))
*/ await workFrame.locator('.search-component-searchBtn').click()
private async processOrder(params: { }
workFrame: FrameLocator
private async openDetailPageFromRow(
workFrame: FrameLocator,
popupPage: Page,
rowIndex: number
): Promise<Page> {
const row = workFrame.locator('tbody tr').nth(rowIndex)
await row.waitFor({ state: 'visible', timeout: 15000 })
const moreButton = row.locator('a.row-more').first()
await moreButton.scrollIntoViewIfNeeded()
const detailPagePromise = popupPage.waitForEvent('popup')
await moreButton.click()
await this.clickMaterialPlanMenu(workFrame)
return await detailPagePromise
}
private async openDetailPageFromCurrentQuery(
workFrame: FrameLocator,
popupPage: Page popupPage: Page
orderNumber: string ): Promise<Page> {
orderIndex: number const firstRow = workFrame.locator('tbody tr').first()
totalOrders: number await firstRow.waitFor({ state: 'visible', timeout: 10000 })
const moreButton = firstRow.locator('a.row-more').first()
const detailPagePromise = popupPage.waitForEvent('popup')
await moreButton.click()
await this.clickMaterialPlanMenu(workFrame)
return await detailPagePromise
}
private async clickMaterialPlanMenu(workFrame: FrameLocator): Promise<void> {
const candidates = [
workFrame.locator('li:visible, a:visible, span:visible, div:visible').filter({
hasText: /^备料计划$/
}),
workFrame.getByRole('menuitem', { name: '备料计划' }),
workFrame.getByText('备料计划', { exact: true }),
workFrame.getByText('备料计划')
]
for (const candidate of candidates) {
const target = candidate.last()
try {
await target.waitFor({ state: 'visible', timeout: 2000 })
await target.click()
return
} catch {
// Try next locator candidate
}
}
throw new Error('无法定位“备料计划”菜单项(可能菜单结构已变化)')
}
private async processDetailPage(params: {
detailPage: Page
deleteSet: Set<string> deleteSet: Set<string>
dryRun: boolean dryRun: boolean
progressState: ProgressState
expectedOrderNumber?: string
onProgress?: ( onProgress?: (
message: string, message: string,
progress?: number, progress?: number,
extra?: Partial<import('../../types/cleaner.types').CleanerProgress> extra?: Partial<import('../../types/cleaner.types').CleanerProgress>
) => void ) => void
}): Promise<OrderCleanDetail> { }): Promise<OrderCleanDetail> {
const { const { detailPage, deleteSet, dryRun, progressState, expectedOrderNumber, onProgress } = params
workFrame,
popupPage,
orderNumber,
orderIndex,
totalOrders,
deleteSet,
dryRun,
onProgress
} = params
const detail: OrderCleanDetail = {
orderNumber,
materialsDeleted: 0,
materialsSkipped: 0,
errors: [],
skippedMaterials: [],
retryCount: 0,
retryAttempts: [],
retriedAt: undefined,
retrySuccess: false
}
// Query the order (Python lines 187-189)
const textbox = workFrame.getByRole('textbox', { name: '生产订单号' })
await textbox.fill(orderNumber)
await workFrame.locator('.search-component-searchBtn').click()
// Wait for loading (Python lines 192-197)
await this.waitForLoading(workFrame)
// Click "更多" to open menu (Python line 200)
await workFrame.locator('#hot-key-head_list').getByText('更多').click()
// Click "备料计划" and expect popup (Python lines 201-203)
const detailPagePromise = popupPage.waitForEvent('popup')
await workFrame.getByText('备料计划').click()
const detailPage = await detailPagePromise
try { try {
// Navigate nested frames in detail page (Python lines 206-207)
const detailMainFrame = detailPage.locator('#forwardFrame') const detailMainFrame = detailPage.locator('#forwardFrame')
const dFrame = await detailMainFrame.contentFrame() const dFrame = await detailMainFrame.contentFrame()
@@ -347,47 +453,53 @@ export class CleanerService {
throw new Error('Failed to access detail inner frame') throw new Error('Failed to access detail inner frame')
} }
// Wait for plan code (Python lines 210-213)
await detailInnerFrame await detailInnerFrame
.getByText(/^离散备料计划维护:/) .getByText(/^离散备料计划维护:/)
.waitFor({ state: 'visible', timeout: 30000 }) .waitFor({ state: 'visible', timeout: 30000 })
// Extract detail count (Python lines 215-218) const sourceOrderNumber = await this.extractSourceOrderNumber(detailInnerFrame)
const orderNumber = sourceOrderNumber || expectedOrderNumber || 'UNKNOWN_ORDER'
const detail: OrderCleanDetail = {
orderNumber,
materialsDeleted: 0,
materialsSkipped: 0,
errors: [],
skippedMaterials: [],
retryCount: 0,
retryAttempts: [],
retriedAt: undefined,
retrySuccess: false
}
const detailCountText = await detailInnerFrame.getByText(/^详细信息 \(\d+\)$/).innerText() const detailCountText = await detailInnerFrame.getByText(/^详细信息 \(\d+\)$/).innerText()
const detailCountMatch = detailCountText.match(/\((\d+)\)/) const detailCountMatch = detailCountText.match(/\((\d+)\)/)
const detailCount = detailCountMatch ? parseInt(detailCountMatch[1], 10) : 0 const detailCount = detailCountMatch ? parseInt(detailCountMatch[1], 10) : 0
// Extract status (Python lines 220-225)
const statusText = await detailInnerFrame.getByText(/^备料状态:.+$/).innerText() const statusText = await detailInnerFrame.getByText(/^备料状态:.+$/).innerText()
const statusMatch = statusText.replace(/\n/g, '').match(/备料状态:(.+)$/) const statusMatch = statusText.replace(/\n/g, '').match(/备料状态:(.+)$/)
const detailStatus = statusMatch ? statusMatch[1].trim() : '' const detailStatus = statusMatch ? statusMatch[1].trim() : ''
// Send progress for order start
onProgress?.( onProgress?.(
`开始处理订单 ${orderIndex + 1}/${totalOrders}: ${orderNumber}`, `开始处理订单: ${orderNumber}`,
((1 + orderIndex) / (1 + totalOrders)) * 100, this.calculateProgress(progressState.completedOrders, 0, detailCount, progressState.totalOrders),
{ {
currentOrderIndex: orderIndex + 1, currentOrderIndex: progressState.completedOrders + 1,
totalOrders, totalOrders: progressState.totalOrders,
currentMaterialIndex: 0, currentMaterialIndex: 0,
totalMaterialsInOrder: detailCount, totalMaterialsInOrder: detailCount,
currentOrderNumber: orderNumber currentOrderNumber: orderNumber
} }
) )
// Process based on status (Python lines 228-441)
if (detailStatus === '审批通过' && detailCount > 0) { if (detailStatus === '审批通过' && detailCount > 0) {
// Click modify button (Python line 235)
await detailInnerFrame.getByRole('button', { name: '修改' }).click() await detailInnerFrame.getByRole('button', { name: '修改' }).click()
// Wait for save button (Python lines 238-242)
const saveButtonLocator = detailInnerFrame.getByRole('button', { name: '保存' }) const saveButtonLocator = detailInnerFrame.getByRole('button', { name: '保存' })
await saveButtonLocator.waitFor({ state: 'visible', timeout: 30000 }) await saveButtonLocator.waitFor({ state: 'visible', timeout: 30000 })
// Expand the form (Python line 245)
await detailInnerFrame.getByText('展开').first().click() await detailInnerFrame.getByText('展开').first().click()
// Get form elements (Python lines 247-253)
const childForm = detailInnerFrame.locator('.card-table-side-box') const childForm = detailInnerFrame.locator('.card-table-side-box')
const buttonWrapper = childForm.locator('.button-wrapper') const buttonWrapper = childForm.locator('.button-wrapper')
const deleteRowBtn = buttonWrapper.getByRole('button', { name: '删行' }) const deleteRowBtn = buttonWrapper.getByRole('button', { name: '删行' })
@@ -397,11 +509,9 @@ export class CleanerService {
let lastRowNumber = '' let lastRowNumber = ''
let materialIdx = 0 let materialIdx = 0
// Process each material row (Python lines 257-423)
while (true) { while (true) {
materialIdx++ materialIdx += 1
// Wait for row number to stabilize (Python lines 262-265)
const currentRow = await this.getInputValue(childForm, /^行号$/) const currentRow = await this.getInputValue(childForm, /^行号$/)
const rowNumInt = parseInt(currentRow, 10) const rowNumInt = parseInt(currentRow, 10)
@@ -409,28 +519,25 @@ export class CleanerService {
await this.delay(500) await this.delay(500)
} }
// Get material data (Python lines 267-271)
const materialCode = await this.getInputValue(childForm, /^材料编码/) const materialCode = await this.getInputValue(childForm, /^材料编码/)
const materialName = await this.getInputValue(childForm, /^材料名称/) const materialName = await this.getInputValue(childForm, /^材料名称/)
const pendingQty = await this.getInputValue(childForm, /^累计待发数量$/) const pendingQty = await this.getInputValue(childForm, /^累计待发数量$/)
// Report progress using formula: (1 + i + j/Mᵢ) / (1 + N) × 100 const progress = this.calculateProgress(
// where i = orderIndex (0-based), j = materialIdx (1-based), Mᵢ = detailCount, N = totalOrders progressState.completedOrders,
const progress = ((1 + orderIndex + materialIdx / detailCount) / (1 + totalOrders)) * 100 materialIdx,
detailCount,
onProgress?.( progressState.totalOrders
`订单 ${orderIndex + 1}/${totalOrders} - 物料 ${materialIdx}/${detailCount}: ${materialName}`,
progress,
{
currentOrderIndex: orderIndex + 1,
totalOrders,
currentMaterialIndex: materialIdx,
totalMaterialsInOrder: detailCount,
currentOrderNumber: orderNumber
}
) )
// Check if should delete (Python lines 284-406) onProgress?.(`订单 ${orderNumber} - 物料 ${materialIdx}/${detailCount}: ${materialName}`, progress, {
currentOrderIndex: progressState.completedOrders + 1,
totalOrders: progressState.totalOrders,
currentMaterialIndex: materialIdx,
totalMaterialsInOrder: detailCount,
currentOrderNumber: orderNumber
})
if (deleteSet.has(materialCode)) { if (deleteSet.has(materialCode)) {
const shouldDelete = this.shouldDeleteMaterial({ const shouldDelete = this.shouldDeleteMaterial({
rowNumber: rowNumInt, rowNumber: rowNumInt,
@@ -440,19 +547,19 @@ export class CleanerService {
}) })
if (shouldDelete && !dryRun) { if (shouldDelete && !dryRun) {
// Delete the material (Python lines 302-340)
const oldRowNumber = currentRow const oldRowNumber = currentRow
await deleteRowBtn.click() await deleteRowBtn.click()
// Wait for row number to change (Python lines 306-324)
const deleteSuccess = await this.waitForRowChange(childForm, oldRowNumber, 10000) const deleteSuccess = await this.waitForRowChange(childForm, oldRowNumber, 10000)
if (deleteSuccess) { if (deleteSuccess) {
detail.materialsDeleted++ detail.materialsDeleted += 1
} }
continue continue
} else if (!shouldDelete) { }
detail.materialsSkipped++
if (!shouldDelete) {
detail.materialsSkipped += 1
const reason = this.getSkipReason({ const reason = this.getSkipReason({
rowNumber: rowNumInt, rowNumber: rowNumInt,
pendingQty, pendingQty,
@@ -468,7 +575,6 @@ export class CleanerService {
} }
} }
// Move to next row (Python lines 419-423)
const isNextEnabled = await this.isButtonEnabled(nextBtn) const isNextEnabled = await this.isButtonEnabled(nextBtn)
if (isNextEnabled) { if (isNextEnabled) {
lastRowNumber = currentRow lastRowNumber = currentRow
@@ -478,27 +584,58 @@ export class CleanerService {
} }
} }
// Collapse form (Python line 424)
await collapseBtn.click() await collapseBtn.click()
// Save changes (Python lines 427-435)
if (!dryRun && detail.materialsDeleted > 0) { if (!dryRun && detail.materialsDeleted > 0) {
await saveButtonLocator.click() await saveButtonLocator.click()
await saveButtonLocator.waitFor({ state: 'hidden', timeout: 60000 }) await saveButtonLocator.waitFor({ state: 'hidden', timeout: 60000 })
} }
} }
return detail
} finally { } finally {
// Close detail page (Python lines 442-443)
await detailPage.close() await detailPage.close()
} }
return detail
} }
/** private calculateProgress(
* Wait for loading overlay to disappear completedOrders: number,
* Reference: Python lines 192-197 materialIdx: number,
*/ detailCount: number,
totalOrders: number
): number {
const materialRatio = detailCount > 0 ? materialIdx / detailCount : 0
return ((1 + completedOrders + materialRatio) / (1 + totalOrders)) * 100
}
private async extractSourceOrderNumber(frame: FrameLocator): Promise<string> {
try {
const sourceOrder = await frame.locator('.vsourcebillcode .code-detail-link').first().innerText()
const match = sourceOrder.match(/SC\d{14}/)
return match ? match[0] : sourceOrder.trim()
} catch {
return ''
}
}
private isOrderNumber(value: string): boolean {
return /^SC\d{14}$/.test(value)
}
private createErrorDetail(orderNumber: string, message: string): OrderCleanDetail {
return {
orderNumber,
materialsDeleted: 0,
materialsSkipped: 0,
errors: [message],
skippedMaterials: [],
retryCount: 0,
retryAttempts: [],
retriedAt: undefined,
retrySuccess: false
}
}
private async waitForLoading(frame: FrameLocator): Promise<void> { private async waitForLoading(frame: FrameLocator): Promise<void> {
const loadingLocator = frame const loadingLocator = frame
.locator('div') .locator('div')
@@ -513,14 +650,7 @@ export class CleanerService {
} }
} }
/** private async getInputValue(container: FrameLocator | Locator, labelRegex: RegExp): Promise<string> {
* Get input value by label regex
* Reference: Python _get_input_value() lines 141-148
*/
private async getInputValue(
container: FrameLocator | Locator,
labelRegex: RegExp
): Promise<string> {
try { try {
return await container return await container
.locator('div') .locator('div')
@@ -533,10 +663,6 @@ export class CleanerService {
} }
} }
/**
* Check if button is enabled
* Reference: Python _is_button_enabled() lines 133-139
*/
private async isButtonEnabled(button: Locator): Promise<boolean> { private async isButtonEnabled(button: Locator): Promise<boolean> {
try { try {
return await button.isEnabled() return await button.isEnabled()
@@ -545,10 +671,6 @@ export class CleanerService {
} }
} }
/**
* Wait for row number to change after deletion
* Reference: Python lines 306-324
*/
private async waitForRowChange( private async waitForRowChange(
childForm: FrameLocator | Locator, childForm: FrameLocator | Locator,
oldRowNumber: string, oldRowNumber: string,
@@ -571,17 +693,10 @@ export class CleanerService {
return false return false
} }
/**
* Delay helper
*/
private delay(ms: number): Promise<void> { private delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms)) return new Promise((resolve) => setTimeout(resolve, ms))
} }
/**
* Retry failed orders from the initial execution
* Maximum 2 retry attempts per failed order
*/
private async retryFailedOrders(params: { private async retryFailedOrders(params: {
workFrame: FrameLocator workFrame: FrameLocator
popupPage: Page popupPage: Page
@@ -610,7 +725,8 @@ export class CleanerService {
const MAX_RETRIES = 2 const MAX_RETRIES = 2
for (const failedDetail of failedDetails) { for (let detailIndex = 0; detailIndex < failedDetails.length; detailIndex++) {
const failedDetail = failedDetails[detailIndex]
const orderNumber = failedDetail.orderNumber const orderNumber = failedDetail.orderNumber
const retryAttempts: import('../../types/cleaner.types').RetryAttempt[] = [] const retryAttempts: import('../../types/cleaner.types').RetryAttempt[] = []
@@ -618,28 +734,25 @@ export class CleanerService {
try { try {
log.info(`Retrying order ${orderNumber} (attempt ${attempt}/${MAX_RETRIES})`) log.info(`Retrying order ${orderNumber} (attempt ${attempt}/${MAX_RETRIES})`)
// Create a new detail for retry await this.queryOrders(workFrame, [orderNumber])
const retryDetail: OrderCleanDetail = { await this.waitForLoading(workFrame)
orderNumber,
materialsDeleted: 0, const rows = workFrame.locator('tbody tr')
materialsSkipped: 0, const rowCount = await rows.count()
errors: [], if (rowCount === 0) {
skippedMaterials: [], throw new Error('订单重试查询无结果')
retryCount: attempt,
retryAttempts: [],
retriedAt: undefined,
retrySuccess: false
} }
// Re-run the order processing const detailPage = await this.openDetailPageFromCurrentQuery(workFrame, popupPage)
await this.processOrder({ const retryDetail = await this.processDetailPage({
workFrame, detailPage,
popupPage,
orderNumber,
orderIndex: 0, // Not used for retry
totalOrders: failedDetails.length,
deleteSet, deleteSet,
dryRun, dryRun,
expectedOrderNumber: orderNumber,
progressState: {
completedOrders: detailIndex,
totalOrders: failedDetails.length
},
onProgress: (message, progress, extra) => { onProgress: (message, progress, extra) => {
onProgress?.( onProgress?.(
`[重试 ${attempt}/${MAX_RETRIES}] ${message}`, `[重试 ${attempt}/${MAX_RETRIES}] ${message}`,
@@ -649,18 +762,16 @@ export class CleanerService {
} }
}) })
// If we reach here, retry succeeded result.successfulRetries += 1
result.successfulRetries++
log.info(`Retry succeeded for order ${orderNumber}`)
// Merge the successful retry detail
result.updatedDetails.push({ result.updatedDetails.push({
...retryDetail, ...retryDetail,
retryCount: attempt,
retriedAt: Date.now(), retriedAt: Date.now(),
retrySuccess: true retrySuccess: true,
retryAttempts
}) })
result.retriedOrders++ result.retriedOrders += 1
break // Exit retry loop for this order break
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error' const message = error instanceof Error ? error.message : 'Unknown error'
log.warn(`Retry attempt ${attempt} failed for order ${orderNumber}: ${message}`) log.warn(`Retry attempt ${attempt} failed for order ${orderNumber}: ${message}`)
@@ -672,8 +783,6 @@ export class CleanerService {
}) })
if (attempt === MAX_RETRIES) { if (attempt === MAX_RETRIES) {
// All retries exhausted
log.error(`All retries failed for order ${orderNumber}`)
result.updatedDetails.push({ result.updatedDetails.push({
...failedDetail, ...failedDetail,
retryCount: MAX_RETRIES, retryCount: MAX_RETRIES,
@@ -681,7 +790,7 @@ export class CleanerService {
retriedAt: Date.now(), retriedAt: Date.now(),
retrySuccess: false retrySuccess: false
}) })
result.retriedOrders++ result.retriedOrders += 1
} }
} }
} }

View File

@@ -16,6 +16,8 @@ export interface CleanerInput {
materialCodes: string[] materialCodes: string[]
dryRun: boolean dryRun: boolean
headless?: boolean headless?: boolean
queryBatchSize?: number
processConcurrency?: number
onProgress?: (message: string, progress?: number, extra?: Partial<CleanerProgress>) => void onProgress?: (message: string, progress?: number, extra?: Partial<CleanerProgress>) => void
} }

View File

@@ -78,6 +78,18 @@ export function useCleaner() {
const saved = sessionStorage.getItem('cleaner_headless') const saved = sessionStorage.getItem('cleaner_headless')
return saved ? saved === 'true' : true return saved ? saved === 'true' : true
}) })
const [queryBatchSize, setQueryBatchSize] = useState(() => {
const saved = sessionStorage.getItem('cleaner_queryBatchSize')
const value = saved ? Number(saved) : 100
if (!Number.isFinite(value)) return 100
return Math.min(100, Math.max(1, Math.trunc(value)))
})
const [processConcurrency, setProcessConcurrency] = useState(() => {
const saved = sessionStorage.getItem('cleaner_processConcurrency')
const value = saved ? Number(saved) : 1
if (!Number.isFinite(value)) return 1
return Math.min(20, Math.max(1, Math.trunc(value)))
})
const [showSettingsMenu, setShowSettingsMenu] = useState(false) const [showSettingsMenu, setShowSettingsMenu] = useState(false)
// Inline editing state for manager field (Admin only) // Inline editing state for manager field (Admin only)
@@ -170,6 +182,14 @@ export function useCleaner() {
sessionStorage.setItem('cleaner_headless', headless.toString()) sessionStorage.setItem('cleaner_headless', headless.toString())
}, [headless]) }, [headless])
useEffect(() => {
sessionStorage.setItem('cleaner_queryBatchSize', queryBatchSize.toString())
}, [queryBatchSize])
useEffect(() => {
sessionStorage.setItem('cleaner_processConcurrency', processConcurrency.toString())
}, [processConcurrency])
useEffect(() => { useEffect(() => {
sessionStorage.setItem('cleaner_validationMode', valMode) sessionStorage.setItem('cleaner_validationMode', valMode)
}, [valMode]) }, [valMode])
@@ -419,7 +439,9 @@ export function useCleaner() {
orderNumbers: orderNumberList, orderNumbers: orderNumberList,
materialCodes: materialCodeList, materialCodes: materialCodeList,
dryRun, dryRun,
headless headless,
queryBatchSize,
processConcurrency
}) })
const cleanerRunData = response.success ? (response.data as any) : null const cleanerRunData = response.success ? (response.data as any) : null
@@ -503,6 +525,10 @@ export function useCleaner() {
setIsTypeDialogOpen, setIsTypeDialogOpen,
headless, headless,
setHeadless, setHeadless,
queryBatchSize,
setQueryBatchSize,
processConcurrency,
setProcessConcurrency,
showSettingsMenu, showSettingsMenu,
setShowSettingsMenu, setShowSettingsMenu,
filteredResults, filteredResults,

View File

@@ -46,6 +46,10 @@ const CleanerPage: React.FC = () => {
setIsTypeDialogOpen, setIsTypeDialogOpen,
headless, headless,
setHeadless, setHeadless,
queryBatchSize,
setQueryBatchSize,
processConcurrency,
setProcessConcurrency,
showSettingsMenu, showSettingsMenu,
setShowSettingsMenu, setShowSettingsMenu,
filteredResults, filteredResults,
@@ -458,6 +462,42 @@ const CleanerPage: React.FC = () => {
</button> </button>
</div> </div>
</div> </div>
<div className="border-t border-slate-100 pt-3 space-y-3">
<div>
<div className="text-sm font-medium text-slate-800"></div>
<div className="text-xs text-slate-500 mt-0.5"> 1-100</div>
<input
type="number"
min={1}
max={100}
value={queryBatchSize}
onChange={(e) => {
const raw = Number(e.target.value)
if (!Number.isFinite(raw)) return
setQueryBatchSize(Math.max(1, Math.min(100, Math.trunc(raw))))
}}
className="mt-2 w-full rounded border border-slate-300 px-2 py-1.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
</div>
<div>
<div className="text-sm font-medium text-slate-800"></div>
<div className="text-xs text-slate-500 mt-0.5">
1-20
</div>
<input
type="number"
min={1}
max={20}
value={processConcurrency}
onChange={(e) => {
const raw = Number(e.target.value)
if (!Number.isFinite(raw)) return
setProcessConcurrency(Math.max(1, Math.min(20, Math.trunc(raw))))
}}
className="mt-2 w-full rounded border border-slate-300 px-2 py-1.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
/>
</div>
</div>
</div> </div>
</div> </div>
)} )}

View File

@@ -1,5 +1,9 @@
import { describe, it, expect } from 'vitest' import { describe, it, expect } from 'vitest'
import { CleanerService } from '../../src/main/services/erp/cleaner' import {
createBatches,
getMissingOrders,
runWithConcurrency
} from '../../src/main/services/erp/cleaner'
import type { ShouldDeleteParams } from '../../src/main/services/erp/cleaner' import type { ShouldDeleteParams } from '../../src/main/services/erp/cleaner'
describe('Cleaner Service (Unit)', () => { describe('Cleaner Service (Unit)', () => {
@@ -132,4 +136,35 @@ describe('Cleaner Service (Unit)', () => {
).toBe(false) ).toBe(false)
}) })
}) })
describe('batch and concurrency helpers', () => {
it('should split orders into batches', () => {
const batches = createBatches(['A', 'B', 'C', 'D', 'E'], 2)
expect(batches).toEqual([
['A', 'B'],
['C', 'D'],
['E']
])
})
it('should identify missing orders', () => {
const missing = getMissingOrders(['SC1', 'SC2', 'SC3'], new Set(['SC1', 'SC3']))
expect(missing).toEqual(['SC2'])
})
it('should respect concurrency limit', async () => {
const items = [1, 2, 3, 4, 5, 6]
let running = 0
let peak = 0
await runWithConcurrency(items, 2, async () => {
running += 1
peak = Math.max(peak, running)
await new Promise((resolve) => setTimeout(resolve, 10))
running -= 1
return true
})
expect(peak).toBeLessThanOrEqual(2)
})
})
}) })

View File

@@ -138,6 +138,34 @@ describe('Cleaner Schema', () => {
expect(result.success).toBe(false) expect(result.success).toBe(false)
}) })
it('should apply defaults for queryBatchSize and processConcurrency', () => {
const input = {
orderNumbers: ['SC12345678901234'],
materialCodes: ['MAT001'],
dryRun: false
}
const result = CleanerInputSchema.safeParse(input)
expect(result.success).toBe(true)
if (result.success) {
expect(result.data.queryBatchSize).toBe(100)
expect(result.data.processConcurrency).toBe(1)
}
})
it('should reject out-of-range processConcurrency', () => {
const input = {
orderNumbers: ['SC12345678901234'],
materialCodes: ['MAT001'],
dryRun: false,
processConcurrency: 21
}
const result = CleanerInputSchema.safeParse(input)
expect(result.success).toBe(false)
})
}) })
describe('validateCleanerInput', () => { describe('validateCleanerInput', () => {