refactor: enhance config handling and unify main entrypoint

Unify configuration parsing logic across entrypoints and clean up legacy script-side config paths.

cmd/go_migrate/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"flag"
 	"sync"
 	"time"
 
@@ -19,7 +20,18 @@ import (
 func main() {
 	configureLog()
 
-	migrationConfig, err := config.ReadMigrationConfig()
+	configPath := flag.String("config", "", "path to migration config file")
+	flag.Parse()
+
+	if flag.NArg() > 1 {
+		log.Fatalf("only one config file path is allowed")
+	}
+
+	if *configPath == "" && flag.NArg() == 1 {
+		*configPath = flag.Arg(0)
+	}
+
+	migrationConfig, err := config.ReadMigrationConfig(*configPath)
 	if err != nil {
 		log.Fatalf("error leyendo configuracion: %v", err)
 	}
@@ -86,7 +98,7 @@ func main() {
 	log.Infof("Migración terminada. Tablas: %d, Errores: %d, Filas totales: %d", len(results), totalErrors, totalProcessed)
 
 	totalDuration := time.Since(startTime)
-	log.Infof("=== Migration completed successfully! ===")
+	// log.Infof("=== Migration completed successfully! ===")
 	log.Infof("Total migration time: %v", totalDuration)
 }
 

cmd/go_migrate/process.go

@@ -13,6 +13,7 @@ import (
 	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractors"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loaders"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
@@ -20,6 +21,8 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
+const jobErrorsChannelSize int = 100
+
 func buildTruncateQuery(targetDbType, schema, table, truncateMethod string) string {
 	if truncateMethod == "DELETE" {
 		if targetDbType == "postgres" {
@@ -41,7 +44,7 @@ func processMigrationJob(
 	targetTableAnalyzer etl.TableAnalyzer,
 	extractor extractors.GenericExtractor,
 	azureClient *azure.Client,
-	loader etl.Loader,
+	loader loaders.GenericLoader,
 	job config.Job,
 	targetDbType string,
 ) models.JobResult {
@@ -54,8 +57,6 @@ func processMigrationJob(
 		StartTime: time.Now(),
 	}
 
-	var rowsRead, rowsLoaded, rowsFailed int64
-
 	var wgQueryColumnTypes errgroup.Group
 	var sourceColTypes, targetColTypes []models.ColumnType
 
@@ -110,17 +111,14 @@ func processMigrationJob(
 		log.Error("Unexpected error calculating batch ranges: ", err)
 	}
 
-	chJobErrors := make(chan custom_errors.JobError, job.ExtractorQueueSize)
-	chLoadersErrors := make(chan custom_errors.LoaderError, job.ExtractorQueueSize)
-	chPartitions := make(chan models.Partition, job.ExtractorQueueSize)
+	chJobErrors := make(chan custom_errors.JobError, jobErrorsChannelSize)
+	chPartitions := make(chan models.Partition)
 	chBatchesRaw := make(chan models.Batch, job.ExtractorQueueSize)
-	chBatchesTransformed := make(chan models.Batch, job.ExtractorQueueSize)
+	chBatchesTransformed := make(chan models.Batch, job.TransformerQueueSize)
 
-	var wgActivePartitions sync.WaitGroup
-	var wgActiveBatches sync.WaitGroup
-	var wgExtractors sync.WaitGroup
-	var wgTransformers sync.WaitGroup
-	var wgLoaders sync.WaitGroup
+	var wgActivePartitions, wgActiveBatches, wgExtractors, wgTransformers, wgLoaders sync.WaitGroup
+	var rowsRead, rowsLoaded, rowsFailed int64
+	var failedPartitionsCount, failedBatchesLoadCount int32
 
 	go func() {
 		if err := custom_errors.JobErrorHandler(localCtx, chJobErrors); err != nil {
@@ -130,18 +128,8 @@ func processMigrationJob(
 		}
 	}()
 
-	go custom_errors.LoaderErrorHandler(
-		localCtx,
-		job.Retry,
-		job.MaxExtractorBatchErrors,
-		chLoadersErrors,
-		chBatchesTransformed,
-		chJobErrors,
-		&wgActiveBatches,
-	)
-
 	maxExtractors := min(job.MaxExtractors, len(partitions))
-	log.Infof("Starting %d extractor(s)...", maxExtractors)
+	log.Infof("Starting %d extractor(s)... (%v)", maxExtractors, job.Name)
 
 	for range maxExtractors {
 		wgExtractors.Go(func() {
@@ -156,6 +144,8 @@ func processMigrationJob(
 				chJobErrors,
 				&wgActivePartitions,
 				&rowsRead,
+				&failedPartitionsCount,
+				job.SourceTable.FromJsonColumns,
 			)
 		})
 	}
@@ -167,13 +157,15 @@ func processMigrationJob(
 		}
 	}()
 
-	log.Infof("Starting %d transformer(s)...", maxExtractors)
+	log.Infof("Starting %d transformer(s)... (%v)", maxExtractors, job.Name)
 
 	for range maxExtractors {
 		wgTransformers.Go(func() {
-			transformer.Exec(
+			transformer.Consume(
 				localCtx,
 				sourceColTypes,
+				job.Retry,
+				job.TransformerBatchSize,
 				chBatchesRaw,
 				chBatchesTransformed,
 				chJobErrors,
@@ -182,48 +174,48 @@ func processMigrationJob(
 		})
 	}
 
-	log.Infof("Starting %d loader(s)...", job.MaxLoaders)
+	log.Infof("Starting %d loader(s)... (%v)", job.MaxLoaders, job.Name)
 
 	for range job.MaxLoaders {
 		wgLoaders.Go(func() {
-			loader.Exec(
+			loader.Consume(
 				localCtx,
 				job.TargetTable,
 				targetColTypes,
+				job.Retry,
+				job.LoaderBatchSize,
 				chBatchesTransformed,
-				chLoadersErrors,
 				chJobErrors,
 				&wgActiveBatches,
 				&rowsLoaded,
+				&failedBatchesLoadCount,
 			)
 		})
 	}
 
 	go func() {
-		log.Debugf("Waiting for goroutines (%v)", job.Name)
+		// log.Debugf("Waiting for goroutines (%v)", job.Name)
 
 		wgActivePartitions.Wait()
-		log.Debugf("wgActivePartitions is empty (%v)", job.Name)
+		// log.Debugf("wgActivePartitions is empty (%v)", job.Name)
 		close(chPartitions)
-		log.Debugf("chPartitions is closed (%v)", job.Name)
+		// log.Debugf("chPartitions is closed (%v)", job.Name)
 
 		wgExtractors.Wait()
-		log.Debugf("wgExtractors is empty (%v)", job.Name)
+		// log.Debugf("wgExtractors is empty (%v)", job.Name)
 		close(chBatchesRaw)
-		log.Debugf("chBatchesRaw is closed (%v)", job.Name)
+		// log.Debugf("chBatchesRaw is closed (%v)", job.Name)
 
 		wgTransformers.Wait()
-		log.Debugf("wgTransformers is empty (%v)", job.Name)
+		// log.Debugf("wgTransformers is empty (%v)", job.Name)
+		close(chBatchesTransformed)
+		// log.Debugf("chBatchesTransformed is closed (%v)", job.Name)
 
 		wgActiveBatches.Wait()
-		log.Debugf("wgActiveBatches is empty (%v)", job.Name)
-		close(chBatchesTransformed)
-		log.Debugf("chBatchesTransformed is empty (%v)", job.Name)
-		close(chLoadersErrors)
-		log.Debugf("chLoadersErrors is empty (%v)", job.Name)
+		// log.Debugf("wgActiveBatches is empty (%v)", job.Name)
 
 		wgLoaders.Wait()
-		log.Debugf("wgLoaders is empty (%v)", job.Name)
+		// log.Debugf("wgLoaders is empty (%v)", job.Name)
 
 		cancel()
 	}()
@@ -235,9 +227,9 @@ func processMigrationJob(
 		}
 	}
 
-	log.Debugf("waiting for local context to be done (%v)", job.Name)
+	// log.Debugf("waiting for local context to be done (%v)", job.Name)
 	<-localCtx.Done()
-	log.Debugf("local context done (%v)", job.Name)
+	// log.Debugf("local context done (%v)", job.Name)
 
 	if ctx.Err() != nil {
 		result.Error = ctx.Err()
@@ -252,5 +244,9 @@ func processMigrationJob(
 		result.Error = fmt.Errorf("Row count mismatch: extracted %d rows but loaded %d rows (failed: %d)", result.RowsRead, result.RowsLoaded, result.RowsFailed)
 	}
 
+	if result.RowsRead == 0 {
+		log.Warnf("No rows extracted from (%v)", job.Name)
+	}
+
 	return result
 }