refactor: enhance config handling and unify main entrypoint

Unify configuration parsing logic across entrypoints and clean up legacy script-side config paths.
2026-05-09 12:00:00 -05:00
parent bf15979c00
commit 6f8a332719
24 changed files with 1177 additions and 575 deletions
--- a/internal/app/etl/loaders/consume.go
+++ b/internal/app/etl/loaders/consume.go
@@ -0,0 +1,154 @@
+package loaders
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+)
+
+func (gl *GenericLoader) Consume(
+	ctx context.Context,
+	tableInfo config.TargetTableInfo,
+	columns []models.ColumnType,
+	retryConfig config.RetryConfig,
+	batchSize int,
+	chBatchesIn <-chan models.Batch,
+	chErrorsOut chan<- custom_errors.JobError,
+	wgActiveBatches *sync.WaitGroup,
+	rowsLoaded *int64,
+	failedBatchesCount *int32,
+) {
+	colNames := mapSlice(columns, func(col models.ColumnType) string {
+		return col.Name()
+	})
+
+	var accRows []models.UnknownRowValues
+	var parentBatchesId []uuid.UUID
+	pendingDone := 0
+
+	defer func() {
+		for range pendingDone {
+			wgActiveBatches.Done()
+		}
+	}()
+
+	flush := func() bool {
+		if len(accRows) == 0 {
+			return true
+		}
+		count := len(parentBatchesId)
+		superBatch := models.Batch{
+			Id:              uuid.New(),
+			ParentBatchesId: parentBatchesId,
+			Rows:            accRows,
+		}
+		processedRows, err := gl.ProcessBatchWithRetries(ctx, tableInfo, colNames, retryConfig, superBatch)
+		for range count {
+			wgActiveBatches.Done()
+		}
+		pendingDone -= count
+		accRows = nil
+		parentBatchesId = nil
+
+		if err != nil {
+			atomic.AddInt32(failedBatchesCount, 1)
+			if jobError, ok := errors.AsType[*custom_errors.JobError](err); ok {
+				select {
+				case <-ctx.Done():
+					return false
+				case chErrorsOut <- *jobError:
+				}
+			} else {
+				select {
+				case <-ctx.Done():
+					return false
+				case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: false, Msg: err.Error(), Prev: err}:
+				}
+			}
+
+			if atomic.LoadInt32(failedBatchesCount) > int32(retryConfig.MaxFailedBatchesLoad) {
+				select {
+				case <-ctx.Done():
+				case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: true, Msg: "Max failed batches (load) reached"}:
+				}
+				return false
+			}
+			return true
+		}
+
+		current := atomic.LoadInt64(rowsLoaded)
+		logrus.Debugf("Rows loaded (batch loaded): +%v [current=%v] (%s.%s)", processedRows, current, tableInfo.Schema, tableInfo.Table)
+		atomic.AddInt64(rowsLoaded, int64(processedRows))
+		return true
+	}
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case batch, ok := <-chBatchesIn:
+			if !ok {
+				flush()
+				return
+			}
+
+			if batchSize <= 0 {
+				processedRows, err := gl.ProcessBatchWithRetries(ctx, tableInfo, colNames, retryConfig, batch)
+				wgActiveBatches.Done()
+
+				if err != nil {
+					atomic.AddInt32(failedBatchesCount, 1)
+					if jobError, ok := errors.AsType[*custom_errors.JobError](err); ok {
+						select {
+						case <-ctx.Done():
+							return
+						case chErrorsOut <- *jobError:
+						}
+					} else {
+						select {
+						case <-ctx.Done():
+							return
+						case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: false, Msg: err.Error(), Prev: err}:
+						}
+					}
+
+					if atomic.LoadInt32(failedBatchesCount) > int32(retryConfig.MaxFailedBatchesLoad) {
+						select {
+						case <-ctx.Done():
+							return
+						case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: true, Msg: "Max failed batches (load) reached"}:
+							return
+						}
+					}
+					continue
+				}
+
+				current := atomic.LoadInt64(rowsLoaded)
+				logrus.Debugf("Rows loaded: +%v [current=%v] (%s.%s)", processedRows, current, tableInfo.Schema, tableInfo.Table)
+				atomic.AddInt64(rowsLoaded, int64(processedRows))
+				continue
+			}
+
+			pendingDone++
+			accRows = append(accRows, batch.Rows...)
+			parentBatchesId = append(parentBatchesId, batch.Id)
+
+			if len(accRows) >= batchSize {
+				if !flush() {
+					return
+				}
+			}
+		}
+	}
+}
--- a/internal/app/etl/loaders/main.go
+++ b/internal/app/etl/loaders/main.go
@@ -1,117 +1,13 @@
 package loaders

 import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
 	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/jackc/pgx/v5/pgconn"
 )

 type GenericLoader struct {
 	db dbwrapper.DbWrapper
 }

-func NewGenericLoader(db dbwrapper.DbWrapper) etl.Loader {
-	return &GenericLoader{db: db}
-}
-
-func (gl *GenericLoader) ProcessBatch(
-	ctx context.Context,
-	tableInfo config.TargetTableInfo,
-	colNames []string,
-	batch models.Batch,
-) (int, error) {
-	_, err := gl.db.SaveMassive(
-		ctx,
-		tableInfo.Schema,
-		tableInfo.Table,
-		colNames,
-		batch.Rows,
-	)
-
-	if err != nil {
-		var pgErr *pgconn.PgError
-		if errors.As(err, &pgErr) {
-			if pgErr.Code == "23505" {
-				return 0, &custom_errors.JobError{
-					ShouldCancelJob: true,
-					Msg:             fmt.Sprintf("Fatal error in table %s.%s", tableInfo.Schema, tableInfo.Table),
-					Prev:            err,
-				}
-			}
-		}
-
-		return 0, &custom_errors.LoaderError{Batch: batch, Msg: err.Error()}
-	}
-
-	return len(batch.Rows), nil
-}
-
-func (gl *GenericLoader) Exec(
-	ctx context.Context,
-	tableInfo config.TargetTableInfo,
-	columns []models.ColumnType,
-	chBatchesIn <-chan models.Batch,
-	chErrorsOut chan<- custom_errors.LoaderError,
-	chJobErrorsOut chan<- custom_errors.JobError,
-	wgActiveBatches *sync.WaitGroup,
-	rowsLoaded *int64,
-) {
-	colNames := mapSlice(columns, func(col models.ColumnType) string {
-		return col.Name()
-	})
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		case batch, ok := <-chBatchesIn:
-			if !ok {
-				return
-			}
-
-			processedRows, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch)
-
-			if err != nil {
-				var ldError *custom_errors.LoaderError
-				var jobError *custom_errors.JobError
-				if errors.As(err, &ldError) {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- *ldError:
-					}
-				} else if errors.As(err, &jobError) {
-					select {
-					case <-ctx.Done():
-						return
-					case chJobErrorsOut <- *jobError:
-					}
-				} else {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- custom_errors.LoaderError{Batch: batch, Msg: err.Error()}:
-					}
-				}
-
-				continue
-			}
-
-			wgActiveBatches.Done()
-			atomic.AddInt64(rowsLoaded, int64(processedRows))
-		}
-	}
+func NewGenericLoader(db dbwrapper.DbWrapper) GenericLoader {
+	return GenericLoader{db: db}
 }
--- a/internal/app/etl/loaders/process-with-retries.go
+++ b/internal/app/etl/loaders/process-with-retries.go
@@ -0,0 +1,49 @@
+package loaders
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+)
+
+func (gl *GenericLoader) ProcessBatchWithRetries(
+	ctx context.Context,
+	tableInfo config.TargetTableInfo,
+	colNames []string,
+	retryConfig config.RetryConfig,
+	batch models.Batch,
+) (int64, error) {
+	for {
+		rowsLoaded, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch)
+		if err == nil {
+			return rowsLoaded, nil
+		}
+
+		if btError, ok := errors.AsType[*custom_errors.LoaderError](err); ok {
+			batch.RetryCounter++
+
+			if batch.RetryCounter >= retryConfig.Attempts {
+				return rowsLoaded, &custom_errors.JobError{
+					Msg:  fmt.Sprintf("Batch %v reached max retries (%d)", batch.Id, batch.RetryCounter),
+					Prev: btError,
+				}
+			}
+
+			delay := custom_errors.ComputeBackoffDelay(
+				batch.RetryCounter,
+				retryConfig.BaseDelayMs,
+				retryConfig.MaxDelayMs,
+				retryConfig.MaxJitterMs,
+			)
+			time.Sleep(delay)
+			continue
+		}
+
+		return rowsLoaded, err
+	}
+}
--- a/internal/app/etl/loaders/process.go
+++ b/internal/app/etl/loaders/process.go
@@ -0,0 +1,43 @@
+package loaders
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/jackc/pgx/v5/pgconn"
+)
+
+func (gl *GenericLoader) ProcessBatch(
+	ctx context.Context,
+	tableInfo config.TargetTableInfo,
+	colNames []string,
+	batch models.Batch,
+) (int64, error) {
+	_, err := gl.db.SaveMassive(
+		ctx,
+		tableInfo.Schema,
+		tableInfo.Table,
+		colNames,
+		batch.Rows,
+	)
+
+	if err != nil {
+		if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
+			if pgErr.Code == "23505" {
+				return 0, &custom_errors.JobError{
+					ShouldCancelJob: true,
+					Msg:             fmt.Sprintf("Fatal error in table %s.%s", tableInfo.Schema, tableInfo.Table),
+					Prev:            err,
+				}
+			}
+		}
+
+		return 0, &custom_errors.LoaderError{Batch: batch, Msg: err.Error()}
+	}
+
+	return int64(len(batch.Rows)), nil
+}