feat(cleaner): add outer-level retry on fatal crash with execution ID
When CleanerService hits a fatal error (browser crash, timeout), the outer catch now sets result.crashed=true. CleanerApplicationService detects this, closes the dead browser session, re-logs into ERP, and re-runs all orders once. An execution ID (CLN-yyyyMMddHHmmss-XXXX) generated at startup ensures report files are deduplicated across retries. Reports now display execution ID and app version. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import type { WebContents } from 'electron'
|
||||
import { app } from 'electron'
|
||||
import type { IDatabaseService } from '../../types/database.types'
|
||||
import { ErpAuthService } from '../erp/erp-auth'
|
||||
import { CleanerService } from '../erp/cleaner'
|
||||
@@ -27,8 +28,21 @@ import type {
|
||||
|
||||
const log = createLogger('CleanerApplicationService')
|
||||
|
||||
function generateExecutionId(): string {
|
||||
const now = new Date()
|
||||
const date = `${now.getFullYear()}${String(now.getMonth() + 1).padStart(2, '0')}${String(now.getDate()).padStart(2, '0')}`
|
||||
const time = `${String(now.getHours()).padStart(2, '0')}${String(now.getMinutes()).padStart(2, '0')}${String(now.getSeconds()).padStart(2, '0')}`
|
||||
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
let suffix = ''
|
||||
for (let i = 0; i < 4; i++) {
|
||||
suffix += chars.charAt(Math.floor(Math.random() * chars.length))
|
||||
}
|
||||
return `CLN-${date}${time}-${suffix}`
|
||||
}
|
||||
|
||||
export class CleanerApplicationService {
|
||||
async runCleaner(eventSender: WebContents, input: CleanerInput): Promise<CleanerResult> {
|
||||
const executionId = generateExecutionId()
|
||||
const startTime = Date.now()
|
||||
let authService: ErpAuthService | null = null
|
||||
let dbService: IDatabaseService | null = null
|
||||
@@ -104,7 +118,6 @@ export class CleanerApplicationService {
|
||||
totalMaterialsInOrder: 0
|
||||
})
|
||||
|
||||
const cleaner = new CleanerService(authService)
|
||||
const modifiedInput: CleanerInput = {
|
||||
...input,
|
||||
orderNumbers: validOrderNumbers,
|
||||
@@ -114,12 +127,68 @@ export class CleanerApplicationService {
|
||||
}
|
||||
|
||||
log.info('Starting cleaning', {
|
||||
executionId,
|
||||
orderCount: validOrderNumbers.length,
|
||||
queryBatchSize: input.queryBatchSize ?? 100,
|
||||
processConcurrency: input.processConcurrency ?? 1
|
||||
})
|
||||
|
||||
const result = await cleaner.clean(modifiedInput)
|
||||
let cleaner = new CleanerService(authService)
|
||||
let result = await cleaner.clean(modifiedInput)
|
||||
|
||||
// Outer retry: re-login and re-run all orders on fatal crash
|
||||
if (result.crashed) {
|
||||
log.warn('检测到流程级崩溃,准备外层重试', { executionId })
|
||||
|
||||
this.sendProgress(eventSender, '流程崩溃,正在重新登录并重试...', 0, {
|
||||
phase: 'retry',
|
||||
currentOrderIndex: 0,
|
||||
totalOrders,
|
||||
currentMaterialIndex: 0,
|
||||
totalMaterialsInOrder: 0
|
||||
})
|
||||
|
||||
try {
|
||||
await authService.close()
|
||||
} catch {
|
||||
// Browser may already be dead, ignore close errors
|
||||
}
|
||||
authService = null
|
||||
|
||||
authService = new ErpAuthService({
|
||||
url: erpConfig.url,
|
||||
username: erpConfig.username,
|
||||
password: erpConfig.password,
|
||||
headless: input.headless ?? true
|
||||
})
|
||||
|
||||
try {
|
||||
await authService.login()
|
||||
log.info('Outer retry: re-login successful', { executionId })
|
||||
} catch (loginError) {
|
||||
log.error('Outer retry: re-login failed', {
|
||||
executionId,
|
||||
error: loginError instanceof Error ? loginError.message : String(loginError)
|
||||
})
|
||||
// Return the original crash result if re-login fails
|
||||
result.errors.push(`外层重试登录失败: ${loginError instanceof Error ? loginError.message : String(loginError)}`)
|
||||
if (warnings.length > 0) {
|
||||
result.errors = [...warnings, ...result.errors]
|
||||
}
|
||||
await this.recordCleanupAudit(validOrderNumbers.length, input, result)
|
||||
await this.generateAndUploadReport(input, result, startTime, executionId)
|
||||
return result
|
||||
}
|
||||
|
||||
cleaner = new CleanerService(authService)
|
||||
result = await cleaner.clean(modifiedInput)
|
||||
log.info('Outer retry completed', {
|
||||
executionId,
|
||||
processedCount: result.ordersProcessed,
|
||||
errorCount: result.errors.length,
|
||||
crashed: result.crashed
|
||||
})
|
||||
}
|
||||
|
||||
if (warnings.length > 0) {
|
||||
result.errors = [...warnings, ...result.errors]
|
||||
@@ -134,12 +203,13 @@ export class CleanerApplicationService {
|
||||
})
|
||||
|
||||
log.info('Cleaning completed', {
|
||||
executionId,
|
||||
processedCount: result.ordersProcessed,
|
||||
errorCount: result.errors.length
|
||||
})
|
||||
|
||||
await this.recordCleanupAudit(validOrderNumbers.length, input, result)
|
||||
await this.generateAndUploadReport(input, result, startTime)
|
||||
await this.generateAndUploadReport(input, result, startTime, executionId)
|
||||
|
||||
return result
|
||||
} finally {
|
||||
@@ -300,7 +370,8 @@ export class CleanerApplicationService {
|
||||
private async generateAndUploadReport(
|
||||
input: CleanerInput,
|
||||
result: CleanerResult,
|
||||
startTime: number
|
||||
startTime: number,
|
||||
executionId: string
|
||||
): Promise<void> {
|
||||
try {
|
||||
const endTime = Date.now()
|
||||
@@ -312,7 +383,9 @@ export class CleanerApplicationService {
|
||||
dryRun: input.dryRun ?? false,
|
||||
username,
|
||||
startTime,
|
||||
endTime
|
||||
endTime,
|
||||
executionId,
|
||||
appVersion: app.getVersion()
|
||||
})
|
||||
log.info('Report generated', { path: reportPath })
|
||||
|
||||
|
||||
@@ -384,6 +384,7 @@ export class CleanerService {
|
||||
...(popupPage ? await capturePageContext(popupPage, undefined, 'cleaner.outerCatch') : {})
|
||||
})
|
||||
result.errors.push(`Clean failed: ${message}`)
|
||||
result.crashed = true
|
||||
} finally {
|
||||
if (popupPage) {
|
||||
try {
|
||||
|
||||
@@ -16,6 +16,8 @@ export interface ReportOptions {
|
||||
username: string
|
||||
startTime: number
|
||||
endTime: number
|
||||
executionId: string
|
||||
appVersion: string
|
||||
}
|
||||
|
||||
interface OrderStats {
|
||||
@@ -41,7 +43,7 @@ export class CleanerReportGenerator {
|
||||
}
|
||||
|
||||
async generateReport(result: CleanerResult, options: ReportOptions): Promise<string> {
|
||||
const filePath = this.getReportFilePath()
|
||||
const filePath = this.getReportFilePath(options.executionId)
|
||||
log.info('Generating cleaner report', { path: filePath })
|
||||
|
||||
const stats = this.calculateOrderStats(result)
|
||||
@@ -53,10 +55,8 @@ export class CleanerReportGenerator {
|
||||
return filePath
|
||||
}
|
||||
|
||||
private getReportFilePath(): string {
|
||||
const now = new Date()
|
||||
const timestamp = now.toISOString().replace(/[:.]/g, '-').slice(0, -5).replace('T', '-')
|
||||
const fileName = `cleaner-report-${timestamp}.md`
|
||||
private getReportFilePath(executionId: string): string {
|
||||
const fileName = `cleaner-report-${executionId}.md`
|
||||
return path.join(this.reportDir, fileName)
|
||||
}
|
||||
|
||||
@@ -87,6 +87,8 @@ export class CleanerReportGenerator {
|
||||
lines.push('')
|
||||
lines.push('| 项目 | 值 |')
|
||||
lines.push('| -------------- | --------------------------------- |')
|
||||
lines.push(`| **执行 ID** | \`${options.executionId}\``)
|
||||
lines.push(`| **应用版本** | \`${options.appVersion}\``)
|
||||
lines.push(`| **执行时间** | \`${this.formatDateTime(options.endTime)}\``)
|
||||
lines.push(`| **执行模式** | \`${options.dryRun ? '模拟运行 (Dry Run)' : '正式执行'}\``)
|
||||
lines.push(`| **操作用户** | \`${options.username}\``)
|
||||
|
||||
Reference in New Issue
Block a user