feat: add exponential backoff retry strategy

Introduce a retry policy with exponential backoff and jitter for extractor and loader errors, with configurable max attempts and delay caps.

cmd/go_migrate/process.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -10,21 +9,24 @@ import (
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/jackc/pgx/v5/pgxpool"
-	_ "github.com/microsoft/go-mssqldb"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 )
 
 func processMigrationJob(
 	ctx context.Context,
-	sourceDb *sql.DB,
-	targetDb *pgxpool.Pool,
+	sourceTableAnalyzer etl.TableAnalyzer,
+	targetTableAnalyzer etl.TableAnalyzer,
 	extractor etl.Extractor,
 	transformer etl.Transformer,
 	loader etl.Loader,
 	job config.Job,
 ) JobResult {
+	jobCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	result := JobResult{
 		JobName:   job.Name,
 		StartTime: time.Now(),
@@ -32,47 +34,85 @@ func processMigrationJob(
 
 	var rowsRead, rowsLoaded, rowsFailed int64
 
-	sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job.SourceTable, job.TargetTable)
+	var wgQueryColumnTypes errgroup.Group
+	var sourceColTypes, targetColTypes []models.ColumnType
+
+	wgQueryColumnTypes.Go(func() error {
+		var err error
+		sourceColTypes, err = sourceTableAnalyzer.QueryColumnTypes(jobCtx, job.SourceTable.TableInfo)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	wgQueryColumnTypes.Go(func() error {
+		var err error
+		targetColTypes, err = targetTableAnalyzer.QueryColumnTypes(jobCtx, job.TargetTable.TableInfo)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	err := wgQueryColumnTypes.Wait()
 	if err != nil {
 		result.Error = err
 		return result
 	}
 
-	logColumnTypes(sourceColTypes, "Source col types")
-	logColumnTypes(targetColTypes, "Target col types")
-
-	jobCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	batches, err := batchGeneratorMssql(jobCtx, sourceDb, job.SourceTable, job.RowsPerBatch)
+	partitions, err := table_analyzers.PartitionRangeGenerator(
+		jobCtx,
+		sourceTableAnalyzer,
+		job.SourceTable.TableInfo,
+		job.SourceTable.PrimaryKey,
+		job.RowsPerPartition,
+	)
 	if err != nil {
 		log.Error("Unexpected error calculating batch ranges: ", err)
 	}
 
 	chJobErrors := make(chan custom_errors.JobError, job.QueueSize)
-	chBatches := make(chan models.Partition, job.QueueSize)
 	chExtractorErrors := make(chan custom_errors.ExtractorError, job.QueueSize)
-	chChunksRaw := make(chan models.Batch, job.QueueSize)
-	chChunksTransformed := make(chan models.Batch, job.QueueSize)
 	chLoadersErrors := make(chan custom_errors.LoaderError, job.QueueSize)
+	chPartitions := make(chan models.Partition, job.QueueSize)
+	chBatchesRaw := make(chan models.Batch, job.QueueSize)
+	chBatchesTransformed := make(chan models.Batch, job.QueueSize)
 
+	var wgActivePartitions sync.WaitGroup
 	var wgActiveBatches sync.WaitGroup
-	var wgActiveChunks sync.WaitGroup
 	var wgExtractors sync.WaitGroup
 	var wgTransformers sync.WaitGroup
 	var wgLoaders sync.WaitGroup
 
 	go func() {
 		if err := custom_errors.JobErrorHandler(jobCtx, chJobErrors); err != nil {
+			log.Error("Fatal error received from JobErrorHandler, canceling job... - ", err)
 			cancel()
 			result.Error = err
 		}
 	}()
 
-	go custom_errors.ExtractorErrorHandler(jobCtx, job.Retry.Attempts, chExtractorErrors, chBatches, chJobErrors, &wgActiveBatches)
-	go custom_errors.LoaderErrorHandler(jobCtx, job.Retry.Attempts, chLoadersErrors, chChunksTransformed, chJobErrors, &wgActiveChunks)
+	go custom_errors.ExtractorErrorHandler(
+		jobCtx,
+		job.Retry,
+		chExtractorErrors,
+		chPartitions,
+		chJobErrors,
+		&wgActivePartitions,
+	)
+	go custom_errors.LoaderErrorHandler(
+		jobCtx,
+		job.Retry,
+		chLoadersErrors,
+		chBatchesTransformed,
+		chJobErrors,
+		&wgActiveBatches,
+	)
 
-	maxExtractors := min(job.MaxExtractors, len(batches))
+	maxExtractors := min(job.MaxExtractors, len(partitions))
 	log.Infof("Starting %d extractor(s)...", maxExtractors)
 
 	for range maxExtractors {
@@ -81,21 +121,21 @@ func processMigrationJob(
 				jobCtx,
 				job.SourceTable,
 				sourceColTypes,
-				job.ChunkSize,
-				chBatches,
-				chChunksRaw,
+				job.BatchSize,
+				chPartitions,
+				chBatchesRaw,
 				chExtractorErrors,
 				chJobErrors,
-				&wgActiveBatches,
+				&wgActivePartitions,
 				&rowsRead,
 			)
 		})
 	}
 
-	wgActiveBatches.Add(len(batches))
+	wgActivePartitions.Add(len(partitions))
 	go func() {
-		for _, batch := range batches {
-			chBatches <- batch
+		for _, batch := range partitions {
+			chPartitions <- batch
 		}
 	}()
 
@@ -106,10 +146,10 @@ func processMigrationJob(
 			transformer.Exec(
 				jobCtx,
 				sourceColTypes,
-				chChunksRaw,
-				chChunksTransformed,
+				chBatchesRaw,
+				chBatchesTransformed,
 				chJobErrors,
-				&wgActiveChunks,
+				&wgActiveBatches,
 			)
 		})
 	}
@@ -122,35 +162,49 @@ func processMigrationJob(
 				jobCtx,
 				job.TargetTable,
 				targetColTypes,
-				chChunksTransformed,
+				chBatchesTransformed,
 				chLoadersErrors,
 				chJobErrors,
-				&wgActiveChunks,
+				&wgActiveBatches,
 				&rowsLoaded,
 			)
 		})
 	}
 
 	go func() {
-		wgActiveBatches.Wait()
-		close(chBatches)
+		log.Debugf("Waiting for goroutines (%v)", job.Name)
+
+		wgActivePartitions.Wait()
+		log.Debugf("wgActivePartitions is empty (%v)", job.Name)
+		close(chPartitions)
+		log.Debugf("chPartitions is closed (%v)", job.Name)
 		close(chExtractorErrors)
+		log.Debugf("chExtractorErrors is closed (%v)", job.Name)
 
 		wgExtractors.Wait()
-		close(chChunksRaw)
+		log.Debugf("wgExtractors is empty (%v)", job.Name)
+		close(chBatchesRaw)
+		log.Debugf("chBatchesRaw is closed (%v)", job.Name)
 
 		wgTransformers.Wait()
+		log.Debugf("wgTransformers is empty (%v)", job.Name)
 
-		wgActiveChunks.Wait()
-		close(chChunksTransformed)
+		wgActiveBatches.Wait()
+		log.Debugf("wgActiveBatches is empty (%v)", job.Name)
+		close(chBatchesTransformed)
+		log.Debugf("chBatchesTransformed is empty (%v)", job.Name)
 		close(chLoadersErrors)
+		log.Debugf("chLoadersErrors is empty (%v)", job.Name)
 
 		wgLoaders.Wait()
+		log.Debugf("wgLoaders is empty (%v)", job.Name)
 
 		cancel()
 	}()
 
+	log.Debugf("waiting for local context to be done (%v)", job.Name)
 	<-jobCtx.Done()
+	log.Debugf("local context done (%v)", job.Name)
 
 	if ctx.Err() != nil {
 		result.Error = ctx.Err()
@@ -163,11 +217,3 @@ func processMigrationJob(
 
 	return result
 }
-
-func logColumnTypes(columnTypes []models.ColumnType, label string) {
-	log.Debug(label)
-
-	for _, col := range columnTypes {
-		log.Debugf("%+v", col)
-	}
-}