perf(import): bypass intermediate Excel file in extraction pipeline
Replace the Extract → Write Excel → Read Excel → Import DB flow with direct record-to-database persistence. The extractor now builds MaterialPlanRecord[] from parsed orders and imports them without the round-trip through a merged Excel file. Key changes: - Add importFromRecords() to DataImportService for record-based import - Add SQL Server OPENJSON batch insert and atomic replace operations in DiscreteMaterialPlanDAO for efficient bulk writes - Extract common import logic into private importRecords() method - Configure explicit request/connection timeouts for SQL Server - Add unit tests for direct record import path Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
104
tests/unit/services/database/discrete-material-plan-dao.test.ts
Normal file
104
tests/unit/services/database/discrete-material-plan-dao.test.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import {
|
||||
DiscreteMaterialPlanDAO,
|
||||
type MaterialPlanRecord
|
||||
} from '../../../../src/main/services/database/discrete-material-plan-dao'
|
||||
import type { IDatabaseService } from '../../../../src/main/services/database'
|
||||
|
||||
vi.mock('../../../../src/main/services/logger', () => ({
|
||||
createLogger: vi.fn(() => ({
|
||||
info: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn()
|
||||
})),
|
||||
getRequestId: vi.fn(() => 'test-request-id'),
|
||||
trackDuration: vi.fn(async (fn) => ({ result: await fn(), durationMs: 1, isSlow: false }))
|
||||
}))
|
||||
|
||||
vi.mock('../../../../src/main/services/database', () => ({
|
||||
create: vi.fn()
|
||||
}))
|
||||
|
||||
function createRecord(sourceNumber: string, index: number): MaterialPlanRecord {
|
||||
return {
|
||||
factory: '工厂A',
|
||||
materialStatus: '已审批',
|
||||
planNumber: `PLAN-${index}`,
|
||||
sourceNumber,
|
||||
materialType: '标准',
|
||||
productCode: 'P001',
|
||||
productName: '产品A',
|
||||
productUnit: 'PCS',
|
||||
productPlanQuantity: 1,
|
||||
useDepartment: '',
|
||||
remark: '',
|
||||
creator: '',
|
||||
createDate: new Date('2026-04-28T00:00:00Z'),
|
||||
approver: '',
|
||||
approveDate: new Date('2026-04-28T00:00:00Z'),
|
||||
sequenceNumber: index,
|
||||
materialCode: `MAT-${index}`,
|
||||
materialName: '物料A',
|
||||
specification: '',
|
||||
model: '',
|
||||
drawingNumber: '',
|
||||
materialQuality: '',
|
||||
planQuantity: 1,
|
||||
unit: 'PCS',
|
||||
requiredDate: new Date('2026-04-28T00:00:00Z'),
|
||||
warehouse: '',
|
||||
unitUsage: 1,
|
||||
cumulativeOutputQuantity: 0,
|
||||
bomVersion: ''
|
||||
}
|
||||
}
|
||||
|
||||
describe('DiscreteMaterialPlanDAO', () => {
|
||||
let mockDbService: IDatabaseService
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks()
|
||||
mockDbService = {
|
||||
type: 'sqlserver',
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
isConnected: vi.fn(() => true),
|
||||
query: vi.fn(async (_sql, params = []) => {
|
||||
const rows = JSON.parse(params[1] || '[]')
|
||||
return {
|
||||
rows: [{ deletedCount: 0, insertedCount: rows.length }],
|
||||
columns: ['deletedCount', 'insertedCount'],
|
||||
rowCount: 1
|
||||
}
|
||||
}),
|
||||
transaction: vi.fn()
|
||||
}
|
||||
|
||||
const database = await import('../../../../src/main/services/database')
|
||||
vi.mocked(database.create).mockResolvedValue(mockDbService)
|
||||
})
|
||||
|
||||
it('splits SQL Server replace operations by source number batches', async () => {
|
||||
const records = Array.from({ length: 151 }, (_, index) =>
|
||||
createRecord(`SC-${String(index).padStart(4, '0')}`, index)
|
||||
)
|
||||
|
||||
const dao = new DiscreteMaterialPlanDAO()
|
||||
const result = await dao.replaceBySourceNumbers(records, 1000)
|
||||
|
||||
expect(result).toEqual({ deleted: 0, inserted: 151 })
|
||||
expect(mockDbService.query).toHaveBeenCalledTimes(7)
|
||||
|
||||
const firstParams = vi.mocked(mockDbService.query).mock.calls[0][1] || []
|
||||
const sixthParams = vi.mocked(mockDbService.query).mock.calls[5][1] || []
|
||||
const seventhParams = vi.mocked(mockDbService.query).mock.calls[6][1] || []
|
||||
|
||||
expect(JSON.parse(firstParams[0])).toHaveLength(25)
|
||||
expect(JSON.parse(sixthParams[0])).toHaveLength(25)
|
||||
expect(JSON.parse(seventhParams[0])).toHaveLength(1)
|
||||
expect(JSON.parse(firstParams[1])).toHaveLength(25)
|
||||
expect(JSON.parse(sixthParams[1])).toHaveLength(25)
|
||||
expect(JSON.parse(seventhParams[1])).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
@@ -97,6 +97,14 @@ describe('ExtractorService', () => {
|
||||
recordsImported: 0,
|
||||
uniqueSourceNumbers: 0,
|
||||
errors: []
|
||||
} as ImportResult),
|
||||
importFromRecords: vi.fn().mockResolvedValue({
|
||||
success: true,
|
||||
recordsRead: 0,
|
||||
recordsDeleted: 0,
|
||||
recordsImported: 0,
|
||||
uniqueSourceNumbers: 0,
|
||||
errors: []
|
||||
} as ImportResult)
|
||||
}
|
||||
|
||||
@@ -173,6 +181,65 @@ describe('ExtractorService', () => {
|
||||
|
||||
expect(Array.isArray(result.errors)).toBe(true)
|
||||
})
|
||||
|
||||
it('should import parsed records directly instead of re-reading merged Excel', async () => {
|
||||
mockExtractorCoreInstance.downloadAllBatches.mockResolvedValue({
|
||||
downloadedFiles: ['./file1.xlsx'],
|
||||
errors: []
|
||||
})
|
||||
mockExcelParserInstance.parse = vi.fn().mockImplementation(() => {
|
||||
mockExcelParserInstance._lastOrders = [
|
||||
{
|
||||
orderInfo: {
|
||||
factory: '工厂A',
|
||||
planNumber: 'PLAN001',
|
||||
productionOrder: 'ORD001',
|
||||
productCode: 'P001',
|
||||
productName: '产品A',
|
||||
plannedQuantity: '10',
|
||||
unit: 'PCS'
|
||||
},
|
||||
materials: [
|
||||
{
|
||||
sequence: 1,
|
||||
materialCode: 'MAT001',
|
||||
materialName: '物料A',
|
||||
quantity: 2,
|
||||
unit: 'PCS'
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
return Promise.resolve()
|
||||
})
|
||||
mockDataImportInstance.importFromRecords.mockResolvedValue({
|
||||
success: true,
|
||||
recordsRead: 1,
|
||||
recordsDeleted: 0,
|
||||
recordsImported: 1,
|
||||
uniqueSourceNumbers: 1,
|
||||
errors: []
|
||||
} as ImportResult)
|
||||
|
||||
const service = new ExtractorService(mockAuthService, './test-downloads')
|
||||
vi.spyOn(service as any, 'saveMergedOrders').mockResolvedValue(undefined)
|
||||
|
||||
const result = await service.extract({
|
||||
orderNumbers: ['ORD001'],
|
||||
onProgress: vi.fn(),
|
||||
onLog: vi.fn()
|
||||
})
|
||||
|
||||
expect(result.importResult?.success).toBe(true)
|
||||
expect(mockDataImportInstance.importFromRecords).toHaveBeenCalledTimes(1)
|
||||
expect(mockDataImportInstance.importFromExcel).not.toHaveBeenCalled()
|
||||
expect(mockDataImportInstance.importFromRecords.mock.calls[0][0][0]).toMatchObject({
|
||||
planNumber: 'PLAN001',
|
||||
sourceNumber: 'ORD001',
|
||||
materialCode: 'MAT001',
|
||||
planQuantity: 2
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('mergeFiles()', () => {
|
||||
@@ -196,6 +263,7 @@ describe('ExtractorService', () => {
|
||||
]
|
||||
|
||||
const service = new ExtractorService(mockAuthService, './test-downloads')
|
||||
vi.spyOn(service as any, 'saveMergedOrders').mockResolvedValue(undefined)
|
||||
|
||||
// @ts-ignore - accessing private method for testing
|
||||
const result = await service.mergeFiles(['./file1.xlsx'], ['ORD001'])
|
||||
@@ -231,6 +299,7 @@ describe('ExtractorService', () => {
|
||||
})
|
||||
|
||||
const service = new ExtractorService(mockAuthService, './test-downloads')
|
||||
vi.spyOn(service as any, 'saveMergedOrders').mockResolvedValue(undefined)
|
||||
|
||||
// @ts-ignore - accessing private method for testing
|
||||
const result = await service.mergeFiles(
|
||||
|
||||
Reference in New Issue
Block a user