refactor: streamline error handling in migration process; consolidate failed partitions and batches tracking

cmd/go_migrate/process.go

@@ -57,8 +57,6 @@ func processMigrationJob(
 		StartTime: time.Now(),
 	}
 
-	var rowsRead, rowsLoaded, rowsFailed int64
-
 	var wgQueryColumnTypes errgroup.Group
 	var sourceColTypes, targetColTypes []models.ColumnType
 
@@ -118,11 +116,9 @@ func processMigrationJob(
 	chBatchesRaw := make(chan models.Batch, job.ExtractorQueueSize)
 	chBatchesTransformed := make(chan models.Batch, job.TransformerQueueSize)
 
-	var wgActivePartitions sync.WaitGroup
-	var wgActiveBatches sync.WaitGroup
-	var wgExtractors sync.WaitGroup
-	var wgTransformers sync.WaitGroup
-	var wgLoaders sync.WaitGroup
+	var wgActivePartitions, wgActiveBatches, wgExtractors, wgTransformers, wgLoaders sync.WaitGroup
+	var rowsRead, rowsLoaded, rowsFailed int64
+	var failedPartitionsCount, failedBatchesLoadCount int32
 
 	go func() {
 		if err := custom_errors.JobErrorHandler(localCtx, chJobErrors); err != nil {
@@ -148,6 +144,7 @@ func processMigrationJob(
 				chJobErrors,
 				&wgActivePartitions,
 				&rowsRead,
+				&failedPartitionsCount,
 				job.SourceTable.FromJsonColumns,
 			)
 		})
@@ -188,6 +185,7 @@ func processMigrationJob(
 				chJobErrors,
 				&wgActiveBatches,
 				&rowsLoaded,
+				&failedBatchesLoadCount,
 			)
 		})
 	}