feat: add retry mechanism with exponential backoff for download failures
This commit is contained in:
@@ -88,6 +88,92 @@ export class DownloadService {
|
|||||||
this.s3Client = this.config.s3Client
|
this.s3Client = this.config.s3Client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute an operation with retry logic using exponential backoff
|
||||||
|
* @param operation - The async operation to execute
|
||||||
|
* @param context - Description of the operation for logging
|
||||||
|
* @param maxRetries - Maximum number of retry attempts (default: 3)
|
||||||
|
* @returns The result of the operation
|
||||||
|
*/
|
||||||
|
private async withRetry<T>(
|
||||||
|
operation: () => Promise<T>,
|
||||||
|
context: string,
|
||||||
|
maxRetries = 3
|
||||||
|
): Promise<T> {
|
||||||
|
let lastError: Error | undefined
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||||
|
try {
|
||||||
|
return await operation()
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error instanceof Error ? error : new Error(String(error))
|
||||||
|
|
||||||
|
// Check if error is retryable
|
||||||
|
const isRetryable = this.isRetryableError(lastError)
|
||||||
|
if (!isRetryable || attempt === maxRetries) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exponential backoff: 1s, 2s, 4s
|
||||||
|
const delay = Math.pow(2, attempt - 1) * 1000
|
||||||
|
log.warn(`${context} failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms`, {
|
||||||
|
error: lastError.message
|
||||||
|
})
|
||||||
|
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delay))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if an error is retryable based on error type and message
|
||||||
|
* @param error - The error to classify
|
||||||
|
* @returns true if the error should trigger a retry
|
||||||
|
*/
|
||||||
|
private isRetryableError(error: Error): boolean {
|
||||||
|
const message = error.message.toLowerCase()
|
||||||
|
const code = (error as any).code?.toLowerCase() || ''
|
||||||
|
|
||||||
|
// Retryable: network errors, timeouts
|
||||||
|
const retryablePatterns = [
|
||||||
|
'etimedout',
|
||||||
|
'econnreset',
|
||||||
|
'timeout',
|
||||||
|
'network',
|
||||||
|
'socket hang up',
|
||||||
|
'connection reset'
|
||||||
|
]
|
||||||
|
|
||||||
|
// Not retryable: S3 errors, validation errors
|
||||||
|
const nonRetryablePatterns = [
|
||||||
|
'404',
|
||||||
|
'403',
|
||||||
|
'not found',
|
||||||
|
'access denied',
|
||||||
|
'invalid',
|
||||||
|
'validation'
|
||||||
|
]
|
||||||
|
|
||||||
|
// Check non-retryable first
|
||||||
|
for (const pattern of nonRetryablePatterns) {
|
||||||
|
if (message.includes(pattern) || code.includes(pattern)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check retryable
|
||||||
|
for (const pattern of retryablePatterns) {
|
||||||
|
if (message.includes(pattern) || code.includes(pattern)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default: don't retry unknown errors
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List all S3 objects under the configured prefix
|
* List all S3 objects under the configured prefix
|
||||||
* Filters to only chromium-* folders
|
* Filters to only chromium-* folders
|
||||||
@@ -130,31 +216,33 @@ export class DownloadService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Download a single file from S3
|
* Download a single file from S3 with retry logic
|
||||||
*/
|
*/
|
||||||
async downloadFile(key: string, destPath: string): Promise<void> {
|
async downloadFile(key: string, destPath: string): Promise<void> {
|
||||||
log.debug('Downloading file', { key, destPath })
|
await this.withRetry(async () => {
|
||||||
|
log.debug('Downloading file', { key, destPath })
|
||||||
|
|
||||||
await fs.promises.mkdir(path.dirname(destPath), { recursive: true })
|
await fs.promises.mkdir(path.dirname(destPath), { recursive: true })
|
||||||
|
|
||||||
const response = await this.s3Client.send(
|
const response = await this.s3Client.send(
|
||||||
new GetObjectCommand({
|
new GetObjectCommand({
|
||||||
Bucket: this.config.bucket,
|
Bucket: this.config.bucket,
|
||||||
Key: key
|
Key: key
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
const output = fs.createWriteStream(destPath)
|
||||||
|
const body = response.Body as NodeJS.ReadableStream
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
body.on('error', reject)
|
||||||
|
output.on('error', reject)
|
||||||
|
output.on('finish', resolve)
|
||||||
|
body.pipe(output)
|
||||||
})
|
})
|
||||||
)
|
|
||||||
|
|
||||||
const output = fs.createWriteStream(destPath)
|
log.debug('File downloaded', { key, destPath })
|
||||||
const body = response.Body as NodeJS.ReadableStream
|
}, `Download ${key}`)
|
||||||
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
body.on('error', reject)
|
|
||||||
output.on('error', reject)
|
|
||||||
output.on('finish', resolve)
|
|
||||||
body.pipe(output)
|
|
||||||
})
|
|
||||||
|
|
||||||
log.debug('File downloaded', { key, destPath })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user