feat: add retry mechanism with exponential backoff for download failures

This commit is contained in:
Misaka_Company
2026-03-24 16:08:38 +08:00
parent fbd62fe390
commit 6f21785c64

View File

@@ -88,6 +88,92 @@ export class DownloadService {
this.s3Client = this.config.s3Client
}
/**
* Execute an operation with retry logic using exponential backoff
* @param operation - The async operation to execute
* @param context - Description of the operation for logging
* @param maxRetries - Maximum number of retry attempts (default: 3)
* @returns The result of the operation
*/
private async withRetry<T>(
operation: () => Promise<T>,
context: string,
maxRetries = 3
): Promise<T> {
let lastError: Error | undefined
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
return await operation()
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error))
// Check if error is retryable
const isRetryable = this.isRetryableError(lastError)
if (!isRetryable || attempt === maxRetries) {
break
}
// Exponential backoff: 1s, 2s, 4s
const delay = Math.pow(2, attempt - 1) * 1000
log.warn(`${context} failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms`, {
error: lastError.message
})
await new Promise((resolve) => setTimeout(resolve, delay))
}
}
throw lastError
}
/**
* Determine if an error is retryable based on error type and message
* @param error - The error to classify
* @returns true if the error should trigger a retry
*/
private isRetryableError(error: Error): boolean {
const message = error.message.toLowerCase()
const code = (error as any).code?.toLowerCase() || ''
// Retryable: network errors, timeouts
const retryablePatterns = [
'etimedout',
'econnreset',
'timeout',
'network',
'socket hang up',
'connection reset'
]
// Not retryable: S3 errors, validation errors
const nonRetryablePatterns = [
'404',
'403',
'not found',
'access denied',
'invalid',
'validation'
]
// Check non-retryable first
for (const pattern of nonRetryablePatterns) {
if (message.includes(pattern) || code.includes(pattern)) {
return false
}
}
// Check retryable
for (const pattern of retryablePatterns) {
if (message.includes(pattern) || code.includes(pattern)) {
return true
}
}
// Default: don't retry unknown errors
return false
}
/**
* List all S3 objects under the configured prefix
* Filters to only chromium-* folders
@@ -130,31 +216,33 @@ export class DownloadService {
}
/**
* Download a single file from S3
* Download a single file from S3 with retry logic
*/
async downloadFile(key: string, destPath: string): Promise<void> {
log.debug('Downloading file', { key, destPath })
await this.withRetry(async () => {
log.debug('Downloading file', { key, destPath })
await fs.promises.mkdir(path.dirname(destPath), { recursive: true })
await fs.promises.mkdir(path.dirname(destPath), { recursive: true })
const response = await this.s3Client.send(
new GetObjectCommand({
Bucket: this.config.bucket,
Key: key
const response = await this.s3Client.send(
new GetObjectCommand({
Bucket: this.config.bucket,
Key: key
})
)
const output = fs.createWriteStream(destPath)
const body = response.Body as NodeJS.ReadableStream
await new Promise<void>((resolve, reject) => {
body.on('error', reject)
output.on('error', reject)
output.on('finish', resolve)
body.pipe(output)
})
)
const output = fs.createWriteStream(destPath)
const body = response.Body as NodeJS.ReadableStream
await new Promise<void>((resolve, reject) => {
body.on('error', reject)
output.on('error', reject)
output.on('finish', resolve)
body.pipe(output)
})
log.debug('File downloaded', { key, destPath })
log.debug('File downloaded', { key, destPath })
}, `Download ${key}`)
}
/**