feat: implement loader error handling and refactor chunk processing in migration job

cmd/go_migrate/process.go

@@ -50,7 +50,7 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
 		extractorErrorHandler(ctx, chExtractorErrors, chBatches, chJobErrors)
 	}()
 
-	chChunks := make(chan Chunk, QueueSize)
+	chChunksRaw := make(chan Chunk, QueueSize)
 	maxExtractors := min(NumExtractors, len(batches))
 	var wgMssqlExtractors sync.WaitGroup
 
@@ -58,7 +58,7 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
 	extractStartTime := time.Now()
 	for range maxExtractors {
 		wgMssqlExtractors.Go(func() {
-			extractFromMssql(ctx, sourceDb, job, sourceColTypes, ChunkSize, chBatches, chChunks, chExtractorErrors, chJobErrors)
+			extractFromMssql(ctx, sourceDb, job, sourceColTypes, ChunkSize, chBatches, chChunksRaw, chExtractorErrors, chJobErrors)
 		})
 	}
 
@@ -72,41 +72,45 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
 
 	go func() {
 		wgMssqlExtractors.Wait()
-		close(chChunks)
+		close(chChunksRaw)
 		log.Infof("Extraction completed in %v", time.Since(extractStartTime))
 	}()
 
-	chChunksTransform := make(chan Chunk, QueueSize)
+	chChunksTransformed := make(chan Chunk, QueueSize)
 	var wgMssqlTransformers sync.WaitGroup
 
 	log.Infof("Starting %d MSSQL transformers...", maxExtractors)
 	transformStartTime := time.Now()
 	for range maxExtractors {
 		wgMssqlTransformers.Go(func() {
-			transformRowsMssql(ctx, sourceColTypes, chChunks, chChunksTransform, chJobErrors)
+			transformRowsMssql(ctx, sourceColTypes, chChunksRaw, chChunksTransformed, chJobErrors)
 		})
 	}
 
 	go func() {
 		wgMssqlTransformers.Wait()
-		close(chChunksTransform)
+		close(chChunksTransformed)
 		log.Infof("Transformation completed in %v", time.Since(transformStartTime))
 	}()
 
 	var wgPostgresLoaders sync.WaitGroup
+	chLoadersErrors := make(chan LoaderError)
+
+	go func() {
+		loaderErrorHandler(ctx, chLoadersErrors, chChunksTransformed, chJobErrors)
+	}()
 
 	log.Infof("Starting %d PostgreSQL loader(s)...", NumLoaders)
 	loaderStartTime := time.Now()
 
 	for range NumLoaders {
 		wgPostgresLoaders.Go(func() {
-			if err := loadRowsPostgres(ctx, job, targetColTypes, targetDb, chChunksTransform); err != nil {
-				log.Error("Unexpected error loading data into postgres: ", err)
-			}
+			loadRowsPostgres(ctx, targetDb, job, targetColTypes, chChunksTransformed, chLoadersErrors)
 		})
 	}
 
 	wgPostgresLoaders.Wait()
+	close(chLoadersErrors)
 	log.Infof("Loading completed in %v", time.Since(loaderStartTime))
 
 	totalDuration := time.Since(jobStartTime)