feat: add retry mechanism with exponential backoff for download failures
This commit is contained in:
@@ -88,6 +88,92 @@ export class DownloadService {
|
||||
this.s3Client = this.config.s3Client
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute an operation with retry logic using exponential backoff
|
||||
* @param operation - The async operation to execute
|
||||
* @param context - Description of the operation for logging
|
||||
* @param maxRetries - Maximum number of retry attempts (default: 3)
|
||||
* @returns The result of the operation
|
||||
*/
|
||||
private async withRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
context: string,
|
||||
maxRetries = 3
|
||||
): Promise<T> {
|
||||
let lastError: Error | undefined
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
return await operation()
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error))
|
||||
|
||||
// Check if error is retryable
|
||||
const isRetryable = this.isRetryableError(lastError)
|
||||
if (!isRetryable || attempt === maxRetries) {
|
||||
break
|
||||
}
|
||||
|
||||
// Exponential backoff: 1s, 2s, 4s
|
||||
const delay = Math.pow(2, attempt - 1) * 1000
|
||||
log.warn(`${context} failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms`, {
|
||||
error: lastError.message
|
||||
})
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, delay))
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if an error is retryable based on error type and message
|
||||
* @param error - The error to classify
|
||||
* @returns true if the error should trigger a retry
|
||||
*/
|
||||
private isRetryableError(error: Error): boolean {
|
||||
const message = error.message.toLowerCase()
|
||||
const code = (error as any).code?.toLowerCase() || ''
|
||||
|
||||
// Retryable: network errors, timeouts
|
||||
const retryablePatterns = [
|
||||
'etimedout',
|
||||
'econnreset',
|
||||
'timeout',
|
||||
'network',
|
||||
'socket hang up',
|
||||
'connection reset'
|
||||
]
|
||||
|
||||
// Not retryable: S3 errors, validation errors
|
||||
const nonRetryablePatterns = [
|
||||
'404',
|
||||
'403',
|
||||
'not found',
|
||||
'access denied',
|
||||
'invalid',
|
||||
'validation'
|
||||
]
|
||||
|
||||
// Check non-retryable first
|
||||
for (const pattern of nonRetryablePatterns) {
|
||||
if (message.includes(pattern) || code.includes(pattern)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Check retryable
|
||||
for (const pattern of retryablePatterns) {
|
||||
if (message.includes(pattern) || code.includes(pattern)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Default: don't retry unknown errors
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* List all S3 objects under the configured prefix
|
||||
* Filters to only chromium-* folders
|
||||
@@ -130,31 +216,33 @@ export class DownloadService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a single file from S3
|
||||
* Download a single file from S3 with retry logic
|
||||
*/
|
||||
async downloadFile(key: string, destPath: string): Promise<void> {
|
||||
log.debug('Downloading file', { key, destPath })
|
||||
await this.withRetry(async () => {
|
||||
log.debug('Downloading file', { key, destPath })
|
||||
|
||||
await fs.promises.mkdir(path.dirname(destPath), { recursive: true })
|
||||
await fs.promises.mkdir(path.dirname(destPath), { recursive: true })
|
||||
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key
|
||||
const response = await this.s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key
|
||||
})
|
||||
)
|
||||
|
||||
const output = fs.createWriteStream(destPath)
|
||||
const body = response.Body as NodeJS.ReadableStream
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
body.on('error', reject)
|
||||
output.on('error', reject)
|
||||
output.on('finish', resolve)
|
||||
body.pipe(output)
|
||||
})
|
||||
)
|
||||
|
||||
const output = fs.createWriteStream(destPath)
|
||||
const body = response.Body as NodeJS.ReadableStream
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
body.on('error', reject)
|
||||
output.on('error', reject)
|
||||
output.on('finish', resolve)
|
||||
body.pipe(output)
|
||||
})
|
||||
|
||||
log.debug('File downloaded', { key, destPath })
|
||||
log.debug('File downloaded', { key, destPath })
|
||||
}, `Download ${key}`)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user