kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: update migration job configuration parameters; rename queue and batch size fields for clarity

Browse Source
This commit is contained in:
Kylesoda
2026-05-05 23:04:27 -05:00
parent 6414943cf3
commit 7cb959a103
3 changed files with 34 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/go_migrate/process.go
View File

@@ -110,11 +110,11 @@ func processMigrationJob(
log.Error("Unexpected error calculating batch ranges: ", err)
}
chJobErrors := make(chan custom_errors.JobError, job.QueueSize)
chLoadersErrors := make(chan custom_errors.LoaderError, job.QueueSize)
chPartitions := make(chan models.Partition, job.QueueSize)
chBatchesRaw := make(chan models.Batch, job.QueueSize)
chBatchesTransformed := make(chan models.Batch, job.QueueSize)
chJobErrors := make(chan custom_errors.JobError, job.ExtractorQueueSize)
chLoadersErrors := make(chan custom_errors.LoaderError, job.ExtractorQueueSize)
chPartitions := make(chan models.Partition, job.ExtractorQueueSize)
chBatchesRaw := make(chan models.Batch, job.ExtractorQueueSize)
chBatchesTransformed := make(chan models.Batch, job.ExtractorQueueSize)
var wgActivePartitions sync.WaitGroup
var wgActiveBatches sync.WaitGroup
@@ -133,7 +133,7 @@ func processMigrationJob(
go custom_errors.LoaderErrorHandler(
localCtx,
job.Retry,
job.MaxChunkErrors,
job.MaxExtractorBatchErrors,
chLoadersErrors,
chBatchesTransformed,
chJobErrors,
@@ -149,7 +149,7 @@ func processMigrationJob(
localCtx,
job.SourceTable,
sourceColTypes,
job.BatchSize,
job.ExtractorBatchSize,
chPartitions,
chBatchesRaw,
chJobErrors,