feat: implement exponential backoff strategy for error handling in extractor and loader processes; enhance retry configuration options

This commit is contained in:
2026-04-12 20:35:29 -05:00
parent 5633dc98d0
commit f126d5bbd0
7 changed files with 122 additions and 23 deletions

View File

@@ -95,8 +95,22 @@ func processMigrationJob(
}
}()
go custom_errors.ExtractorErrorHandler(jobCtx, job.Retry.Attempts, chExtractorErrors, chPartitions, chJobErrors, &wgActivePartitions)
go custom_errors.LoaderErrorHandler(jobCtx, job.Retry.Attempts, chLoadersErrors, chBatchesTransformed, chJobErrors, &wgActiveBatches)
go custom_errors.ExtractorErrorHandler(
jobCtx,
job.Retry,
chExtractorErrors,
chPartitions,
chJobErrors,
&wgActivePartitions,
)
go custom_errors.LoaderErrorHandler(
jobCtx,
job.Retry,
chLoadersErrors,
chBatchesTransformed,
chJobErrors,
&wgActiveBatches,
)
maxExtractors := min(job.MaxExtractors, len(partitions))
log.Infof("Starting %d extractor(s)...", maxExtractors)