refactor: remove extractor error channel and simplify retry logic in mssql and postgres extractors
This commit is contained in:
@@ -85,7 +85,6 @@ func processMigrationJob(
|
||||
}
|
||||
|
||||
chJobErrors := make(chan custom_errors.JobError, job.QueueSize)
|
||||
chExtractorErrors := make(chan custom_errors.ExtractorError, job.QueueSize)
|
||||
chLoadersErrors := make(chan custom_errors.LoaderError, job.QueueSize)
|
||||
chPartitions := make(chan models.Partition, job.QueueSize)
|
||||
chBatchesRaw := make(chan models.Batch, job.QueueSize)
|
||||
@@ -105,15 +104,6 @@ func processMigrationJob(
|
||||
}
|
||||
}()
|
||||
|
||||
go custom_errors.ExtractorErrorHandler(
|
||||
localCtx,
|
||||
job.Retry,
|
||||
job.MaxPartitionErrrors,
|
||||
chExtractorErrors,
|
||||
chPartitions,
|
||||
chJobErrors,
|
||||
&wgActivePartitions,
|
||||
)
|
||||
go custom_errors.LoaderErrorHandler(
|
||||
localCtx,
|
||||
job.Retry,
|
||||
@@ -136,7 +126,6 @@ func processMigrationJob(
|
||||
job.BatchSize,
|
||||
chPartitions,
|
||||
chBatchesRaw,
|
||||
chExtractorErrors,
|
||||
chJobErrors,
|
||||
&wgActivePartitions,
|
||||
&rowsRead,
|
||||
@@ -190,8 +179,6 @@ func processMigrationJob(
|
||||
log.Debugf("wgActivePartitions is empty (%v)", job.Name)
|
||||
close(chPartitions)
|
||||
log.Debugf("chPartitions is closed (%v)", job.Name)
|
||||
close(chExtractorErrors)
|
||||
log.Debugf("chExtractorErrors is closed (%v)", job.Name)
|
||||
|
||||
wgExtractors.Wait()
|
||||
log.Debugf("wgExtractors is empty (%v)", job.Name)
|
||||
|
||||
Reference in New Issue
Block a user