feat: add configuration parsing and job settings structure for YAML migration configuration

.gitignore

@@ -4,7 +4,6 @@
 *.dll
 *.so
 *.dylib
-bin/
 
 # Test binary, built with `go test -c`
 *.test

Makefile

@@ -1,80 +0,0 @@
-.PHONY: build build-linux build-windows build-all clean help
-
-# Variables
-BINARY_NAME=go-migrate
-CMD_PATH=./cmd/go_migrate
-OUTPUT_DIR=bin
-VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
-BUILD_TIME=$(shell date -u '+%Y-%m-%d_%H:%M:%S')
-GIT_COMMIT=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
-
-# Flags de compilación
-LD_FLAGS=-ldflags="-s -w -X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT)"
-
-# Default: compilar para el SO actual
-build: build-$(OS)
-
-ifeq ($(OS),Windows_NT)
-build-native: build-windows
-else
-build-native: build-linux
-endif
-
-# Compilar para Linux (sin CGO para máxima compatibilidad)
-build-linux:
-	@echo "Compilando para Linux..."
-	@mkdir -p $(OUTPUT_DIR)
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-		$(LD_FLAGS) \
-		-o $(OUTPUT_DIR)/$(BINARY_NAME)-linux-amd64 \
-		$(CMD_PATH)
-	@echo "Binario creado: $(OUTPUT_DIR)/$(BINARY_NAME)-linux-amd64"
-
-# Compilar para Windows
-build-windows:
-	@echo "Compilando para Windows..."
-	@mkdir -p $(OUTPUT_DIR)
-	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build \
-		$(LD_FLAGS) \
-		-o $(OUTPUT_DIR)/$(BINARY_NAME)-windows-amd64.exe \
-		$(CMD_PATH)
-	@echo "Binario creado: $(OUTPUT_DIR)/$(BINARY_NAME)-windows-amd64.exe"
-
-# Compilar para ambas plataformas
-build-all: build-linux build-windows
-	@echo ""
-	@echo "Binarios compilados:"
-	@ls -lh $(OUTPUT_DIR)/$(BINARY_NAME)*
-
-# Compilar para Linux arm64 (opcional, para Raspberry Pi, etc.)
-build-linux-arm64:
-	@echo "Compilando para Linux ARM64..."
-	@mkdir -p $(OUTPUT_DIR)
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build \
-		$(LD_FLAGS) \
-		-o $(OUTPUT_DIR)/$(BINARY_NAME)-linux-arm64 \
-		$(CMD_PATH)
-	@echo "Binario creado: $(OUTPUT_DIR)/$(BINARY_NAME)-linux-arm64"
-
-# Limpiar binarios
-clean:
-	@echo "Limpiando binarios..."
-	@rm -rf $(OUTPUT_DIR)
-	@echo "Limpieza completada"
-
-# Ayuda
-help:
-	@echo "Comandos disponibles:"
-	@echo ""
-	@echo "  make build              - Compilar para el SO actual (Linux/Windows)"
-	@echo "  make build-linux        - Compilar para Linux x86_64"
-	@echo "  make build-windows      - Compilar para Windows x86_64"
-	@echo "  make build-linux-arm64  - Compilar para Linux ARM64 (opcional)"
-	@echo "  make build-all          - Compilar para Linux y Windows"
-	@echo "  make clean              - Eliminar binarios compilados"
-	@echo "  make help               - Mostrar esta ayuda"
-	@echo ""
-	@echo "Ejemplos de uso:"
-	@echo "  make build-all          # Crear binarios para ambas plataformas"
-	@echo "  make build-linux OS=    # Crear solo para Linux"
-	@echo ""

cmd/go_migrate/batch-generator.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+type Batch struct {
+	Id                    uuid.UUID
+	ParentId              uuid.UUID
+	LowerLimit            int64
+	UpperLimit            int64
+	IsLowerLimitInclusive bool
+	ShouldUseRange        bool
+	RetryCounter          int
+}
+
+func estimateTotalRowsMssql(ctx context.Context, db *sql.DB, job MigrationJob) (int64, error) {
+	query := `
+SELECT
+	SUM(p.rows) AS count
+FROM sys.tables t
+JOIN sys.schemas s ON t.schema_id = s.schema_id
+JOIN sys.partitions p ON t.object_id = p.object_id
+WHERE s.name = @schema AND t.name = @table AND p.index_id IN (0, 1)
+GROUP BY t.name`
+
+	ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*20)
+	defer cancel()
+
+	var rowsCount int64
+	err := db.QueryRowContext(ctxTimeout, query, sql.Named("schema", job.Schema), sql.Named("table", job.Table)).Scan(&rowsCount)
+	if err != nil {
+		return 0, err
+	}
+
+	return rowsCount, nil
+}
+
+func calculateBatchesMssql(ctx context.Context, db *sql.DB, job MigrationJob, batchCount int64) ([]Batch, error) {
+	query := fmt.Sprintf(`
+SELECT
+	MIN([%s]) AS lower_limit,
+	MAX([%s]) AS upper_limit
+FROM
+	(SELECT [%s], NTILE(@batchCount) OVER (ORDER BY [%s]) AS batch_id FROM [%s].[%s]) AS T
+GROUP BY batch_id
+ORDER BY batch_id`, job.PrimaryKey, job.PrimaryKey, job.PrimaryKey, job.PrimaryKey, job.Schema, job.Table)
+
+	ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*20)
+	defer cancel()
+
+	rows, err := db.QueryContext(ctxTimeout, query, sql.Named("batchCount", batchCount))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	batches := make([]Batch, 0, batchCount)
+
+	for rows.Next() {
+		batch := Batch{
+			Id:                    uuid.New(),
+			ShouldUseRange:        true,
+			RetryCounter:          0,
+			IsLowerLimitInclusive: true,
+		}
+
+		if err := rows.Scan(&batch.LowerLimit, &batch.UpperLimit); err != nil {
+			return nil, err
+		}
+
+		batches = append(batches, batch)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return batches, nil
+}
+
+func batchGeneratorMssql(ctx context.Context, db *sql.DB, job MigrationJob) ([]Batch, error) {
+	rowsCount, err := estimateTotalRowsMssql(ctx, db, job)
+	if err != nil {
+		return nil, err
+	}
+
+	var batchCount int64 = 1
+	if rowsCount > RowsPerBatch {
+		batchCount = rowsCount / RowsPerBatch
+	} else {
+		return []Batch{{
+			Id:             uuid.New(),
+			ShouldUseRange: false,
+			RetryCounter:   0,
+		}}, nil
+	}
+
+	batches, err := calculateBatchesMssql(ctx, db, job, batchCount)
+	if err != nil {
+		return nil, err
+	}
+
+	return batches, nil
+}

cmd/go_migrate/build-extract-query.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+func buildExtractQueryMssql(job MigrationJob, columns []ColumnType, includeRange bool, isMinInclusive bool) string {
+	var sbQuery strings.Builder
+
+	sbQuery.WriteString("SELECT ")
+
+	if len(columns) == 0 {
+		sbQuery.WriteString("*")
+	} else {
+		for i, col := range columns {
+			fmt.Fprintf(&sbQuery, "[%s]", col.name)
+
+			if col.unifiedType == "GEOMETRY" {
+				fmt.Fprintf(&sbQuery, ".STAsBinary() AS [%s]", col.name)
+			}
+
+			if i < len(columns)-1 {
+				sbQuery.WriteString(", ")
+			}
+		}
+	}
+
+	fmt.Fprintf(&sbQuery, " FROM [%s].[%s]", job.Schema, job.Table)
+
+	if includeRange {
+		fmt.Fprintf(&sbQuery, " WHERE [%s]", job.PrimaryKey)
+		if isMinInclusive {
+			sbQuery.WriteString(" >=")
+		} else {
+			sbQuery.WriteString(" >")
+		}
+
+		fmt.Fprintf(&sbQuery, " @min AND [%s] <= @max", job.PrimaryKey)
+	}
+
+	fmt.Fprintf(&sbQuery, " ORDER BY [%s] ASC", job.PrimaryKey)
+
+	return sbQuery.String()
+}
+
+func buildExtractQueryPostgres(job MigrationJob, columns []ColumnType) string {
+	var sbColumns strings.Builder
+
+	if len(columns) == 0 {
+		sbColumns.WriteString("*")
+	} else {
+		for i, col := range columns {
+			if col.unifiedType == "GEOMETRY" {
+				sbColumns.WriteString(`ST_AsEWKB("`)
+				sbColumns.WriteString(col.name)
+				sbColumns.WriteString(`") AS "`)
+				sbColumns.WriteString(col.name)
+				sbColumns.WriteString(`"`)
+			} else {
+				sbColumns.WriteString(`"`)
+				sbColumns.WriteString(col.name)
+				sbColumns.WriteString(`"`)
+			}
+
+			if i < len(columns)-1 {
+				sbColumns.WriteString(", ")
+			}
+		}
+	}
+
+	return fmt.Sprintf(`SELECT %s FROM "%s"."%s"  ORDER BY "%s" ASC`, sbColumns.String(), job.Schema, job.Table, job.PrimaryKey)
+}

cmd/go_migrate/colum-type.go

@@ -1,4 +1,4 @@
-package models
+package main
 
 type ColumnType struct {
 	name string
@@ -42,29 +42,3 @@ func (c *ColumnType) Nullable() bool {
 func (c *ColumnType) Type() string {
 	return c.unifiedType
 }
-
-func NewColumnType(
-	name string,
-	hasMaxLength bool,
-	hasPrecisionScale bool,
-	userType string,
-	systemType string,
-	unifiedType string,
-	nullable bool,
-	maxLength int64,
-	precision int64,
-	scale int64,
-) ColumnType {
-	return ColumnType{
-		name,
-		hasMaxLength,
-		hasPrecisionScale,
-		userType,
-		systemType,
-		unifiedType,
-		nullable,
-		maxLength,
-		precision,
-		scale,
-	}
-}

cmd/go_migrate/extractor-error-handler.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/google/uuid"
+)
+
+type ExtractorError struct {
+	Batch
+	LastId    int64
+	HasLastId bool
+	Msg       string
+}
+
+func (e *ExtractorError) Error() string {
+	return e.Msg
+}
+
+const maxRetryAttempts = 3
+
+func extractorErrorHandler(
+	ctx context.Context,
+	chErrorsIn <-chan ExtractorError,
+	chBatchesOut chan<- Batch,
+	chJobErrorsOut chan<- JobError,
+	wgActiveBatches *sync.WaitGroup,
+) {
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+
+		case err, ok := <-chErrorsIn:
+			if !ok {
+				return
+			}
+
+			if err.RetryCounter >= maxRetryAttempts {
+				jobError := JobError{
+					ShouldCancelJob: false,
+					Msg:             fmt.Sprintf("batch %v reached max retries (%d)", err.Id, maxRetryAttempts),
+					Prev:            &err,
+				}
+
+				select {
+				case chJobErrorsOut <- jobError:
+				case <-ctx.Done():
+					return
+				}
+
+				wgActiveBatches.Done()
+				continue
+			}
+
+			newBatch := err.Batch
+			newBatch.RetryCounter++
+
+			if err.HasLastId {
+				newBatch.ParentId = err.Id
+				newBatch.Id = uuid.New()
+				newBatch.LowerLimit = err.LastId
+				newBatch.IsLowerLimitInclusive = false
+			}
+
+			select {
+			case chBatchesOut <- newBatch:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}
+}
+
+func ExtractorErrorFromLastRowMssql(lastRow UnknownRowValues, indexPrimaryKey int, batch *Batch, previousError error) ExtractorError {
+	lastIdRawValue := lastRow[indexPrimaryKey]
+
+	lastId, ok := ToInt64(lastIdRawValue)
+	if !ok {
+		currentBatch := *batch
+		currentBatch.RetryCounter = maxRetryAttempts
+		return ExtractorError{
+			Batch:     currentBatch,
+			HasLastId: true,
+			Msg:       fmt.Sprintf("Couldn't cast last id value as int: %s", previousError.Error()),
+		}
+
+	}
+
+	return ExtractorError{
+		Batch:     *batch,
+		HasLastId: true,
+		LastId:    lastId,
+		Msg:       previousError.Error(),
+	}
+}

cmd/go_migrate/extractor.go

@@ -0,0 +1,242 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+	_ "github.com/microsoft/go-mssqldb"
+	log "github.com/sirupsen/logrus"
+)
+
+type UnknownRowValues = []any
+
+type Chunk struct {
+	Id           uuid.UUID
+	BatchId      uuid.UUID
+	Data         []UnknownRowValues
+	RetryCounter int
+}
+
+func extractFromMssql(
+	ctx context.Context,
+	db *sql.DB,
+	job MigrationJob,
+	columns []ColumnType,
+	chunkSize int,
+	chBatchesIn <-chan Batch,
+	chChunksOut chan<- Chunk,
+	chErrorsOut chan<- ExtractorError,
+	chJobErrorsOut chan<- JobError,
+	wgActiveBatches *sync.WaitGroup,
+) {
+	indexPrimaryKey := slices.IndexFunc(columns, func(col ColumnType) bool {
+		return strings.EqualFold(col.name, job.PrimaryKey)
+	})
+
+	if indexPrimaryKey == -1 {
+		jobError := JobError{
+			ShouldCancelJob: true,
+			Msg:             "Primary key not found in provided columns",
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case chJobErrorsOut <- jobError:
+		}
+
+		return
+	}
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case batch, ok := <-chBatchesIn:
+			if !ok {
+				return
+			}
+
+			if abort := processBatch(ctx, db, job, columns, chunkSize, batch, indexPrimaryKey, chChunksOut, chErrorsOut, wgActiveBatches); abort {
+				return
+			}
+		}
+	}
+}
+
+func processBatch(
+	ctx context.Context,
+	db *sql.DB,
+	job MigrationJob,
+	columns []ColumnType,
+	chunkSize int,
+	batch Batch,
+	indexPrimaryKey int,
+	chChunksOut chan<- Chunk,
+	chErrorsOut chan<- ExtractorError,
+	wgActiveBatches *sync.WaitGroup,
+) (abort bool) {
+	query := buildExtractQueryMssql(job, columns, batch.ShouldUseRange, batch.IsLowerLimitInclusive)
+	log.Debug("Query used to extract data from mssql: ", query)
+
+	var queryArgs []any
+	if batch.ShouldUseRange {
+		queryArgs = append(queryArgs,
+			sql.Named("min", batch.LowerLimit),
+			sql.Named("max", batch.UpperLimit),
+		)
+	}
+
+	queryStartTime := time.Now()
+	rows, err := db.QueryContext(ctx, query, queryArgs...)
+	if err != nil {
+		select {
+		case chErrorsOut <- ExtractorError{Batch: batch, HasLastId: false, Msg: err.Error()}:
+		case <-ctx.Done():
+			return true
+		}
+		return false
+	}
+	defer rows.Close()
+	log.Debugf("Query executed in %v", time.Since(queryStartTime))
+
+	rowsChunk := make([]UnknownRowValues, 0, chunkSize)
+	totalRowsExtracted := 0
+	chunkStartTime := time.Now()
+
+	for rows.Next() {
+		values := make([]any, len(columns))
+		scanArgs := make([]any, len(columns))
+
+		for i := range values {
+			scanArgs[i] = &values[i]
+		}
+
+		if err := rows.Scan(scanArgs...); err != nil {
+			if len(rowsChunk) == 0 {
+				select {
+				case chErrorsOut <- ExtractorError{Batch: batch, HasLastId: false, Msg: err.Error()}:
+				case <-ctx.Done():
+					return true
+				}
+				return false
+			}
+
+			lastRow := rowsChunk[len(rowsChunk)-1]
+			select {
+			case chErrorsOut <- ExtractorErrorFromLastRowMssql(lastRow, indexPrimaryKey, &batch, err):
+			case <-ctx.Done():
+				return true
+			}
+
+			select {
+			case chChunksOut <- Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
+			case <-ctx.Done():
+				return true
+			}
+
+			return false
+		}
+
+		rowsChunk = append(rowsChunk, values)
+		totalRowsExtracted++
+
+		if len(rowsChunk) >= chunkSize {
+			chunkDuration := time.Since(chunkStartTime)
+			rowsPerSec := float64(chunkSize) / chunkDuration.Seconds()
+			log.Infof("Extracted chunk: %d rows in %v (%.0f rows/sec) - Total: %d rows", len(rowsChunk), chunkDuration, rowsPerSec, totalRowsExtracted)
+
+			select {
+			case chChunksOut <- Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
+			case <-ctx.Done():
+				return true
+			}
+
+			rowsChunk = make([]UnknownRowValues, 0, chunkSize)
+			chunkStartTime = time.Now()
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		if errors.Is(err, ctx.Err()) {
+			return true
+		}
+
+		if len(rowsChunk) == 0 {
+			select {
+			case chErrorsOut <- ExtractorError{Batch: batch, HasLastId: false, Msg: err.Error()}:
+			case <-ctx.Done():
+				return true
+			}
+			return false
+		}
+
+		lastRow := rowsChunk[len(rowsChunk)-1]
+		select {
+		case chErrorsOut <- ExtractorErrorFromLastRowMssql(lastRow, indexPrimaryKey, &batch, err):
+		case <-ctx.Done():
+			return true
+		}
+		return false
+	}
+
+	if len(rowsChunk) > 0 {
+		chunkDuration := time.Since(chunkStartTime)
+		rowsPerSec := float64(len(rowsChunk)) / chunkDuration.Seconds()
+		log.Infof("Extracted final chunk: %d rows in %v (%.0f rows/sec) - Total: %d rows", len(rowsChunk), chunkDuration, rowsPerSec, totalRowsExtracted)
+		select {
+		case chChunksOut <- Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
+		case <-ctx.Done():
+			return true
+		}
+	}
+
+	wgActiveBatches.Done()
+	return false
+}
+
+func extractFromPostgres(ctx context.Context, job MigrationJob, columns []ColumnType, chunkSize int, db *pgxpool.Pool, out chan<- []UnknownRowValues) error {
+	query := buildExtractQueryPostgres(job, columns)
+	log.Debug("Query used to extract data from postgres: ", query)
+
+	rows, err := db.Query(ctx, query)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	rowsChunk := make([]UnknownRowValues, 0, chunkSize)
+
+	for rows.Next() {
+		values, err := rows.Values()
+		if err != nil {
+			return err
+		}
+
+		rowsChunk = append(rowsChunk, values)
+
+		if len(rowsChunk) >= chunkSize {
+			out <- rowsChunk
+			rowsChunk = make([]UnknownRowValues, 0, chunkSize)
+			log.Infof("Chunk send... %+v", job)
+		}
+	}
+
+	if len(rowsChunk) > 0 {
+		out <- rowsChunk
+		log.Infof("Chunk send... %+v", job)
+	}
+
+	return nil
+}

cmd/go_migrate/inspect-columns.go

@@ -0,0 +1,283 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	_ "github.com/microsoft/go-mssqldb"
+	log "github.com/sirupsen/logrus"
+)
+
+func GetUnifiedType(systemType string) string {
+	systemType = strings.ToLower(systemType)
+
+	if systemType == "varchar" || systemType == "char" || systemType == "nvarchar" || systemType == "nchar" || systemType == "text" || systemType == "ntext" {
+		return "STRING"
+	}
+
+	if systemType == "int" || systemType == "int4" || systemType == "integer" || systemType == "smallint" || systemType == "int2" || systemType == "bigint" || systemType == "int8" || systemType == "tinyint" {
+		return "INTEGER"
+	}
+
+	if systemType == "decimal" || systemType == "numeric" {
+		return "DECIMAL"
+	}
+
+	if systemType == "float" || systemType == "real" || systemType == "double precision" {
+		return "FLOAT"
+	}
+
+	if systemType == "bit" || systemType == "boolean" {
+		return "BOOLEAN"
+	}
+
+	if systemType == "date" {
+		return "DATE"
+	}
+	if systemType == "time" || systemType == "time without time zone" {
+		return "TIME"
+	}
+	if systemType == "datetime" || systemType == "datetime2" || systemType == "timestamp" || systemType == "timestamptz" || systemType == "timestamp with time zone" {
+		return "TIMESTAMP"
+	}
+
+	if systemType == "binary" || systemType == "varbinary" || systemType == "image" || systemType == "bytea" {
+		return "BINARY"
+	}
+
+	if systemType == "uniqueidentifier" || systemType == "uuid" {
+		return "UUID"
+	}
+
+	if systemType == "json" {
+		return "JSON"
+	}
+
+	if systemType == "geometry" || systemType == "geography" {
+		return "GEOMETRY"
+	}
+
+	return strings.ToUpper(systemType)
+}
+
+func MapPostgresColumn(column ColumnType, maxLength *int64, precision *int64, scale *int64) ColumnType {
+	stringTypes := map[string]bool{
+		"varchar": true, "char": true, "character": true, "text": true, "character varying": true,
+	}
+
+	decimalTypes := map[string]bool{
+		"decimal": true, "numeric": true,
+	}
+
+	if stringTypes[column.systemType] {
+		if maxLength != nil {
+			column.maxLength = *maxLength
+			column.hasMaxLength = true
+		} else {
+			column.maxLength = -1
+			column.hasMaxLength = false
+		}
+		column.hasPrecisionScale = false
+		column.precision = -1
+		column.scale = -1
+	} else if decimalTypes[column.systemType] {
+		column.hasMaxLength = false
+		column.maxLength = -1
+		if precision != nil && scale != nil {
+			column.precision = *precision
+			column.scale = *scale
+			column.hasPrecisionScale = true
+		} else {
+			column.precision = -1
+			column.scale = -1
+			column.hasPrecisionScale = false
+		}
+	} else {
+		column.hasMaxLength = false
+		column.maxLength = -1
+		column.hasPrecisionScale = false
+		column.precision = -1
+		column.scale = -1
+	}
+
+	column.unifiedType = GetUnifiedType(column.systemType)
+
+	return column
+}
+
+func GetColumnTypesPostgres(db *pgxpool.Pool, migrationJob MigrationJob) ([]ColumnType, error) {
+	query := `
+SELECT 
+  c.column_name AS name,
+  c.data_type AS user_type,
+  c.udt_name AS system_type,
+  (CASE WHEN c.is_nullable = 'YES' THEN TRUE ELSE FALSE END) AS nullable,
+  c.character_maximum_length AS max_length,
+  c.numeric_precision AS precision,
+  c.numeric_scale AS scale
+FROM information_schema.columns c
+WHERE c.table_schema = $1 AND c.table_name = $2
+ORDER BY c.ordinal_position;
+`
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	rows, err := db.Query(ctx, query, migrationJob.Schema, migrationJob.Table)
+	if err != nil {
+		return nil, fmt.Errorf("Error querying column types: %w", err)
+	}
+	defer rows.Close()
+
+	var colTypes []ColumnType
+
+	for rows.Next() {
+		var column ColumnType
+		var scanMaxLength *int64
+		var scanPrecision *int64
+		var scanScale *int64
+
+		if err := rows.Scan(
+			&column.name,
+			&column.userType,
+			&column.systemType,
+			&column.nullable,
+			&scanMaxLength,
+			&scanPrecision,
+			&scanScale,
+		); err != nil {
+			return nil, fmt.Errorf("Error scanning column type results: %w", err)
+		}
+
+		colTypes = append(colTypes, MapPostgresColumn(column, scanMaxLength, scanPrecision, scanScale))
+	}
+
+	return colTypes, nil
+}
+
+func MapMssqlColumn(column ColumnType) ColumnType {
+	stringTypes := map[string]bool{
+		"varchar": true, "char": true, "nvarchar": true, "nchar": true, "text": true, "ntext": true,
+	}
+
+	decimalTypes := map[string]bool{
+		"decimal": true, "numeric": true,
+	}
+
+	if stringTypes[column.systemType] {
+		column.hasMaxLength = true
+		if column.systemType == "nvarchar" || column.systemType == "nchar" {
+			if column.maxLength > 0 {
+				column.maxLength = column.maxLength / 2
+			}
+		}
+		column.hasPrecisionScale = false
+		column.precision = -1
+		column.scale = -1
+	} else if decimalTypes[column.systemType] {
+		column.hasMaxLength = false
+		column.maxLength = -1
+		column.hasPrecisionScale = true
+	} else {
+		column.hasMaxLength = false
+		column.maxLength = -1
+		column.hasPrecisionScale = false
+		column.precision = -1
+		column.scale = -1
+	}
+
+	column.unifiedType = GetUnifiedType(column.systemType)
+
+	return column
+}
+
+func GetColumnTypesMssql(db *sql.DB, migrationJob MigrationJob) ([]ColumnType, error) {
+	query := `
+SELECT 
+	c.name AS name,
+	t.name AS user_type,
+	CASE WHEN t.is_user_defined = 0 THEN t.name ELSE bt.name END AS system_type,
+	c.is_nullable AS nullable,
+	c.max_length AS max_length,
+	c.precision AS precision,
+	c.scale AS scale
+FROM sys.columns c
+JOIN sys.types t ON c.user_type_id = t.user_type_id
+LEFT JOIN sys.types bt ON t.is_user_defined = 1 AND bt.user_type_id = t.system_type_id
+JOIN sys.tables st ON c.object_id = st.object_id
+JOIN sys.schemas s ON st.schema_id = s.schema_id
+WHERE s.name = @schema AND st.name = @table
+ORDER BY c.column_id;
+`
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	rows, err := db.QueryContext(ctx, query, sql.Named("schema", migrationJob.Schema), sql.Named("table", migrationJob.Table))
+	if err != nil {
+		return nil, fmt.Errorf("Error querying column types: %w", err)
+	}
+	defer rows.Close()
+
+	var colTypes []ColumnType
+
+	for rows.Next() {
+		var column ColumnType
+
+		if err := rows.Scan(
+			&column.name,
+			&column.userType,
+			&column.systemType,
+			&column.nullable,
+			&column.maxLength,
+			&column.precision,
+			&column.scale,
+		); err != nil {
+			return nil, fmt.Errorf("Error scanning column type results: %W", err)
+		}
+
+		if strings.HasPrefix(column.name, "graph_id") && column.systemType == "bigint" {
+			continue
+		}
+
+		colTypes = append(colTypes, MapMssqlColumn(column))
+	}
+
+	return colTypes, nil
+}
+
+func GetColumnTypes(sourceDb *sql.DB, targetDb *pgxpool.Pool, migrationJob MigrationJob) ([]ColumnType, []ColumnType, error) {
+	var sourceDbErr error
+	var targetDbErr error
+	var sourceColTypes []ColumnType
+	var targetColTypes []ColumnType
+	var wg sync.WaitGroup
+
+	wg.Go(func() {
+		sourceColTypes, sourceDbErr = GetColumnTypesMssql(sourceDb, migrationJob)
+		if sourceDbErr != nil {
+			log.Error("Error (sourceDb): ", sourceDbErr)
+		}
+	})
+
+	wg.Go(func() {
+		targetColTypes, targetDbErr = GetColumnTypesPostgres(targetDb, migrationJob)
+		if targetDbErr != nil {
+			log.Error("Error (targetDb): ", targetDbErr)
+		}
+	})
+
+	wg.Wait()
+
+	if sourceDbErr != nil || targetDbErr != nil {
+		return nil, nil, errors.New("Error querying column types")
+	}
+
+	return sourceColTypes, targetColTypes, nil
+}

cmd/go_migrate/job-error-handler.go

@@ -1,4 +1,4 @@
-package custom_errors
+package main
 
 import (
 	"context"
@@ -21,7 +21,7 @@ func (e *JobError) Error() string {
 	return e.Msg
 }
 
-func JobErrorHandler(ctx context.Context, chErrorsIn <-chan JobError) error {
+func jobErrorHandler(ctx context.Context, chErrorsIn <-chan JobError) error {
 	for {
 		if ctx.Err() != nil {
 			return nil
@@ -37,11 +37,11 @@ func JobErrorHandler(ctx context.Context, chErrorsIn <-chan JobError) error {
 			}
 
 			if err.ShouldCancelJob {
-				log.Errorf("(Fatal job error) - %v - %v", err.Msg, err.Prev)
+				log.Error(err.Msg, " - ", err.Prev)
 				return &err
 			}
 
-			log.Errorf("%v - %v", err.Msg, err.Prev)
+			log.Error(err.Msg, " - ", err.Prev)
 		}
 	}
 }

cmd/go_migrate/loader-error-handler.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sync"
+)
+
+type LoaderError struct {
+	Chunk
+	Msg string
+}
+
+func (e *LoaderError) Error() string {
+	return e.Msg
+}
+
+func loaderErrorHandler(
+	ctx context.Context,
+	chErrorsIn <-chan LoaderError,
+	chChunksOut chan<- Chunk,
+	chJobErrorsOut chan<- JobError,
+	wgActiveChunks *sync.WaitGroup,
+) {
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+
+		case err, ok := <-chErrorsIn:
+			if !ok {
+				return
+			}
+
+			if err.RetryCounter >= maxRetryAttempts {
+				jobError := JobError{
+					ShouldCancelJob: false,
+					Msg:             fmt.Sprintf("chunk %v reached max retries (%d)", err.Id, maxRetryAttempts),
+					Prev:            &err,
+				}
+
+				select {
+				case chJobErrorsOut <- jobError:
+				case <-ctx.Done():
+					return
+				}
+
+				wgActiveChunks.Done()
+				continue
+			}
+
+			err.RetryCounter++
+
+			select {
+			case chChunksOut <- err.Chunk:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}
+}

cmd/go_migrate/loader.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/jackc/pgx/v5/pgxpool"
+	mssql "github.com/microsoft/go-mssqldb"
+	log "github.com/sirupsen/logrus"
+)
+
+func loadRowsPostgres(
+	ctx context.Context,
+	db *pgxpool.Pool,
+	job MigrationJob,
+	columns []ColumnType,
+	chChunksIn <-chan Chunk,
+	chErrorsOut chan<- LoaderError,
+	chJobErrorsOut chan<- JobError,
+	wgActiveChunks *sync.WaitGroup,
+) {
+	tableId := pgx.Identifier{job.Schema, job.Table}
+	colNames := Map(columns, func(col ColumnType) string {
+		return col.name
+	})
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case chunk, ok := <-chChunksIn:
+			if !ok {
+				return
+			}
+
+			if abort := loadChunkPostgres(ctx, db, tableId, colNames, chunk, chErrorsOut, chJobErrorsOut, wgActiveChunks); abort {
+				return
+			}
+		}
+	}
+}
+
+func loadChunkPostgres(
+	ctx context.Context,
+	db *pgxpool.Pool,
+	identifier pgx.Identifier,
+	colNames []string,
+	chunk Chunk,
+	chErrorsOut chan<- LoaderError,
+	chJobErrorsOut chan<- JobError,
+	wgActiveChunks *sync.WaitGroup,
+) (abort bool) {
+	chunkStartTime := time.Now()
+	_, err := db.CopyFrom(
+		ctx,
+		identifier,
+		colNames,
+		pgx.CopyFromRows(chunk.Data),
+	)
+
+	if err != nil {
+		var pgErr *pgconn.PgError
+		if errors.As(err, &pgErr) {
+			if pgErr.Code == "23505" {
+				select {
+				case chJobErrorsOut <- JobError{
+					ShouldCancelJob: true,
+					Msg:             fmt.Sprintf("Fatal data integrity error in table %s", identifier.Sanitize()),
+					Prev:            err,
+				}:
+				case <-ctx.Done():
+				}
+				wgActiveChunks.Done()
+				return true
+			}
+		}
+
+		select {
+		case chErrorsOut <- LoaderError{Chunk: chunk, Msg: err.Error()}:
+		case <-ctx.Done():
+			return true
+		}
+		return false
+	}
+
+	chunkDuration := time.Since(chunkStartTime)
+	rowsPerSec := float64(len(chunk.Data)) / chunkDuration.Seconds()
+
+	log.Infof("Loaded chunk: %d rows in %v (%.0f rows/sec)", len(chunk.Data), chunkDuration, rowsPerSec)
+
+	wgActiveChunks.Done()
+	return false
+}
+
+func loadRowsMssql(ctx context.Context, job MigrationJob, columns []ColumnType, db *sql.DB, in <-chan []UnknownRowValues) error {
+	chunkCount := 0
+	totalRowsLoaded := 0
+
+	for rows := range in {
+		chunkStartTime := time.Now()
+
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			return fmt.Errorf("error starting transaction: %w", err)
+		}
+
+		fullTableName := fmt.Sprintf("[%s].[%s]", job.Schema, job.Table)
+		colNames := Map(columns, func(col ColumnType) string {
+			return col.name
+		})
+
+		stmt, err := tx.PrepareContext(ctx, mssql.CopyIn(fullTableName, mssql.BulkOptions{}, colNames...))
+		if err != nil {
+			tx.Rollback()
+			return fmt.Errorf("error preparing bulk copy statement: %w", err)
+		}
+
+		copyStartTime := time.Now()
+
+		for _, row := range rows {
+			_, err = stmt.ExecContext(ctx, row...)
+			if err != nil {
+				stmt.Close()
+				tx.Rollback()
+				return fmt.Errorf("error executing row insert: %w", err)
+			}
+		}
+
+		result, err := stmt.ExecContext(ctx)
+		if err != nil {
+			stmt.Close()
+			tx.Rollback()
+			return fmt.Errorf("error flushing bulk data: %w", err)
+		}
+
+		err = stmt.Close()
+		if err != nil {
+			tx.Rollback()
+			return fmt.Errorf("error closing statement: %w", err)
+		}
+
+		if err := tx.Commit(); err != nil {
+			return fmt.Errorf("error committing transaction: %w", err)
+		}
+
+		rowsAffected, _ := result.RowsAffected()
+		chunkCount++
+		totalRowsLoaded += int(rowsAffected)
+
+		copyDuration := time.Since(copyStartTime)
+		chunkDuration := time.Since(chunkStartTime)
+		rowsPerSec := float64(len(rows)) / chunkDuration.Seconds()
+
+		log.Infof("Loaded chunk #%d (MSSQL): %d rows in %v (copy: %v, %.0f rows/sec) - Total: %d rows", chunkCount, len(rows), chunkDuration, copyDuration, rowsPerSec, totalRowsLoaded)
+	}
+
+	return nil
+}
+
+func Map[T any, V any](input []T, mapper func(T) V) []V {
+	result := make([]V, len(input))
+
+	for i, v := range input {
+		result[i] = mapper(v)
+	}
+
+	return result
+}
+
+func fakeLoader(job MigrationJob, columns []ColumnType, in <-chan [][]any) {
+
+	for rows := range in {
+		log.Debugf("Chunk received, loading data into...")
+
+		for i, rowValues := range rows {
+			if i%100 == 0 {
+				logSampleRow(job, columns, rowValues, fmt.Sprintf("row %d", i))
+			}
+		}
+	}
+}

cmd/go_migrate/log.go

@@ -13,5 +13,5 @@ func configureLog() {
 		DisableSorting:  false,
 		PadLevelText:    true,
 	})
-	log.SetLevel(log.DebugLevel)
+	log.SetLevel(log.InfoLevel)
 }

cmd/go_migrate/main.go

@@ -2,175 +2,63 @@ package main
 
 import (
 	"context"
-	"sync"
 	"time"
 
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loaders"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformers"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
+)
+
+type MigrationJob struct {
+	Schema     string
+	Table      string
+	PrimaryKey string
+}
+
+var migrationJobs []MigrationJob = []MigrationJob{
+	{
+		Schema:     "Cartografia",
+		Table:      "MANZANA",
+		PrimaryKey: "GDB_ARCHIVE_OID",
+	},
+	{
+		Schema:     "Red",
+		Table:      "PUERTO",
+		PrimaryKey: "ID_PUERTO",
+	},
+}
+
+const (
+	NumExtractors  int   = 4
+	NumLoaders     int   = 8
+	ChunkSize      int   = 25000
+	QueueSize      int   = 8
+	ChunksPerBatch int   = 16
+	RowsPerBatch   int64 = int64(ChunkSize * ChunksPerBatch)
 )
 
 func main() {
 	configureLog()
-
-	migrationConfig, err := config.ReadMigrationConfig()
-	if err != nil {
-		log.Fatalf("error leyendo configuracion: %v", err)
-	}
-
-	log.Debugf("Config: %+v", migrationConfig)
-
 	startTime := time.Now()
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	log.Info("=== Starting migration ===")
+	log.Infof("Number of loaders: %d, Chunk size: %d", NumLoaders, ChunkSize)
 
-	var wgConnect errgroup.Group
-	var sourceDb, targetDb dbwrapper.DbWrapper
-
-	wgConnect.Go(func() error {
-		var err error
-		sourceDb, err = connectWithTimeout(ctx, migrationConfig.SourceDbType, config.App.SourceDbUrl, 20*time.Second)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	wgConnect.Go(func() error {
-		var err error
-		targetDb, err = connectWithTimeout(ctx, migrationConfig.TargetDbType, config.App.TargetDbUrl, 20*time.Second)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	if err := wgConnect.Wait(); err != nil {
-		log.Error("Connection error: ", err)
-		return
+	sourceDb, targetDb, connError := connectToDatabases()
+	if connError != nil {
+		log.Fatal("Connection error: ", connError)
 	}
+
 	defer sourceDb.Close()
 	defer targetDb.Close()
 
-	results := processMigrationJobs(ctx, sourceDb, targetDb, migrationConfig.Jobs, migrationConfig.MaxParallelWorkers)
-
-	log.Info("=== RESUMEN DE MIGRACIÓN ===")
-	var totalProcessed, totalErrors int64
-
-	for _, res := range results {
-		status := "OK"
-		if res.Error != nil {
-			status = "FAILED"
-			log.Infof("[%s] Status: %s | Read: %d | Loaded: %d | Errors: %d | Time: %v | Error: %v", res.JobName, status, res.RowsRead, res.RowsLoaded, res.RowsFailed, res.Duration, res.Error)
-		} else {
-			log.Infof("[%s] Status: %s | Read: %d | Loaded: %d | Errors: %d | Time: %v", res.JobName, status, res.RowsRead, res.RowsLoaded, res.RowsFailed, res.Duration)
-		}
-
-		totalProcessed += res.RowsLoaded
-		if res.Error != nil {
-			totalErrors++
-		}
+	for _, job := range migrationJobs {
+		log.Infof(">>> Processing job: %s.%s <<<", job.Schema, job.Table)
+		processMigrationJob(ctx, sourceDb, targetDb, job)
 	}
 
-	log.Infof("Migración terminada. Tablas: %d, Errores: %d, Filas totales: %d", len(results), totalErrors, totalProcessed)
-
 	totalDuration := time.Since(startTime)
 	log.Infof("=== Migration completed successfully! ===")
 	log.Infof("Total migration time: %v", totalDuration)
 }
-
-func processMigrationJobs(
-	ctx context.Context,
-	sourceDb dbwrapper.DbWrapper,
-	targetDb dbwrapper.DbWrapper,
-	jobs []config.Job,
-	maxParallelWorkers int,
-) []JobResult {
-	if len(jobs) == 0 {
-		log.Info("No migration jobs configured")
-		return []JobResult{}
-	}
-
-	if maxParallelWorkers <= 0 {
-		maxParallelWorkers = 1
-	}
-
-	if maxParallelWorkers > len(jobs) {
-		maxParallelWorkers = len(jobs)
-	}
-
-	log.Infof("Starting migration with %d parallel worker(s)", maxParallelWorkers)
-
-	chJobResults := make(chan JobResult, len(jobs))
-	chJobs := make(chan config.Job, len(jobs))
-	var wgJobs sync.WaitGroup
-
-	sourceTableAnalyzer := table_analyzers.NewMssqlTableAnalyzer(sourceDb)
-	targetTableAnalyzer := table_analyzers.NewPostgresTableAnalyzer(targetDb)
-	extractor := extractors.NewMssqlExtractor(sourceDb)
-	transformer := transformers.NewMssqlTransformer()
-	loader := loaders.NewPostgresLoader(targetDb)
-
-	for i := range maxParallelWorkers {
-		wgJobs.Go(func() {
-			for job := range chJobs {
-				log.Infof("[worker %d] >>> Processing job: %s.%s <<<", i, job.SourceTable.Schema, job.SourceTable.Table)
-				res := processMigrationJob(
-					ctx,
-					targetDb,
-					sourceTableAnalyzer,
-					targetTableAnalyzer,
-					extractor,
-					transformer,
-					loader,
-					job,
-				)
-
-				chJobResults <- res
-			}
-		})
-	}
-
-	for _, job := range jobs {
-		chJobs <- job
-	}
-	close(chJobs)
-
-	go func() {
-		wgJobs.Wait()
-		close(chJobResults)
-	}()
-
-	var finalResults []JobResult
-	for res := range chJobResults {
-		finalResults = append(finalResults, res)
-	}
-
-	return finalResults
-}
-
-func connectWithTimeout(ctx context.Context, dbType string, dbUrl string, timeout time.Duration) (dbwrapper.DbWrapper, error) {
-	localCtx, cancel := context.WithTimeout(ctx, timeout)
-	defer cancel()
-
-	sourceDb, err := dbwrapper.New(dbType)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = sourceDb.Connect(localCtx, dbUrl); err != nil {
-		return nil, err
-	}
-
-	return sourceDb, nil
-}

cmd/go_migrate/metrics.go

@@ -1,13 +0,0 @@
-package main
-
-import "time"
-
-type JobResult struct {
-	JobName    string
-	StartTime  time.Time
-	Duration   time.Duration
-	RowsRead   int64
-	RowsLoaded int64
-	RowsFailed int64
-	Error      error
-}

cmd/go_migrate/mssql-transform.go

@@ -1,4 +1,4 @@
-package transformers
+package main
 
 import (
 	"encoding/binary"

cmd/go_migrate/process.go

@@ -2,230 +2,135 @@ package main
 
 import (
 	"context"
-	"fmt"
+	"database/sql"
 	"sync"
-	"sync/atomic"
 	"time"
 
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	_ "github.com/microsoft/go-mssqldb"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
 )
 
 func processMigrationJob(
 	ctx context.Context,
-	targetDbWrapper dbwrapper.DbWrapper,
-	sourceTableAnalyzer etl.TableAnalyzer,
-	targetTableAnalyzer etl.TableAnalyzer,
-	extractor etl.Extractor,
-	transformer etl.Transformer,
-	loader etl.Loader,
-	job config.Job,
-) JobResult {
-	localCtx, cancel := context.WithCancel(ctx)
+	sourceDb *sql.DB,
+	targetDb *pgxpool.Pool,
+	job MigrationJob,
+) {
+	jobStartTime := time.Now()
+	log.Infof("Starting migration job: %s.%s [PK: %s]", job.Schema, job.Table, job.PrimaryKey)
+
+	sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job)
+	if err != nil {
+		log.Fatal("Unexpected error: ", err)
+	}
+
+	logColumnTypes(sourceColTypes, "Source col types")
+	logColumnTypes(targetColTypes, "Target col types")
+
+	jobCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	result := JobResult{
-		JobName:   job.Name,
-		StartTime: time.Now(),
-	}
-
-	var rowsRead, rowsLoaded, rowsFailed int64
-
-	var wgQueryColumnTypes errgroup.Group
-	var sourceColTypes, targetColTypes []models.ColumnType
-
-	wgQueryColumnTypes.Go(func() error {
-		var err error
-		sourceColTypes, err = sourceTableAnalyzer.QueryColumnTypes(localCtx, job.SourceTable.TableInfo)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	wgQueryColumnTypes.Go(func() error {
-		var err error
-		targetColTypes, err = targetTableAnalyzer.QueryColumnTypes(localCtx, job.TargetTable.TableInfo)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	err := wgQueryColumnTypes.Wait()
-	if err != nil {
-		result.Error = err
-		return result
-	}
-
-	for _, query := range job.PreSQL {
-		if _, err := targetDbWrapper.Exec(localCtx, query); err != nil {
-			result.Error = err
-			return result
-		}
-	}
-
-	partitions, err := table_analyzers.PartitionRangeGenerator(
-		localCtx,
-		sourceTableAnalyzer,
-		job.SourceTable.TableInfo,
-		job.SourceTable.PrimaryKey,
-		job.RowsPerPartition,
-	)
+	batches, err := batchGeneratorMssql(jobCtx, sourceDb, job)
 	if err != nil {
 		log.Error("Unexpected error calculating batch ranges: ", err)
 	}
 
-	chJobErrors := make(chan custom_errors.JobError, job.QueueSize)
-	chLoadersErrors := make(chan custom_errors.LoaderError, job.QueueSize)
-	chPartitions := make(chan models.Partition, job.QueueSize)
-	chBatchesRaw := make(chan models.Batch, job.QueueSize)
-	chBatchesTransformed := make(chan models.Batch, job.QueueSize)
+	chJobErrors := make(chan JobError, 50)
+	chBatches := make(chan Batch, QueueSize)
+	chExtractorErrors := make(chan ExtractorError, QueueSize)
+	chChunksRaw := make(chan Chunk, QueueSize)
+	chChunksTransformed := make(chan Chunk, QueueSize)
+	chLoadersErrors := make(chan LoaderError, QueueSize)
 
-	var wgActivePartitions sync.WaitGroup
 	var wgActiveBatches sync.WaitGroup
+	var wgActiveChunks sync.WaitGroup
 	var wgExtractors sync.WaitGroup
 	var wgTransformers sync.WaitGroup
 	var wgLoaders sync.WaitGroup
 
 	go func() {
-		if err := custom_errors.JobErrorHandler(localCtx, chJobErrors); err != nil {
-			log.Error("Fatal error received from JobErrorHandler, canceling job... - ", err)
+		if err := jobErrorHandler(jobCtx, chJobErrors); err != nil {
 			cancel()
-			result.Error = err
 		}
 	}()
 
-	go custom_errors.LoaderErrorHandler(
-		localCtx,
-		job.Retry,
-		job.MaxChunkErrors,
-		chLoadersErrors,
-		chBatchesTransformed,
-		chJobErrors,
-		&wgActiveBatches,
-	)
+	go extractorErrorHandler(jobCtx, chExtractorErrors, chBatches, chJobErrors, &wgActiveBatches)
+	go loaderErrorHandler(jobCtx, chLoadersErrors, chChunksTransformed, chJobErrors, &wgActiveChunks)
 
-	maxExtractors := min(job.MaxExtractors, len(partitions))
-	log.Infof("Starting %d extractor(s)...", maxExtractors)
+	maxExtractors := min(NumExtractors, len(batches))
+	log.Infof("Starting %d extractors...", maxExtractors)
+	extractStartTime := time.Now()
 
 	for range maxExtractors {
 		wgExtractors.Go(func() {
-			extractors.Consume(
-				localCtx,
-				extractor,
-				job.SourceTable,
-				sourceColTypes,
-				job.BatchSize,
-				chPartitions,
-				chBatchesRaw,
-				chJobErrors,
-				&wgActivePartitions,
-				&rowsRead,
-			)
+			extractFromMssql(jobCtx, sourceDb, job, sourceColTypes, ChunkSize, chBatches, chChunksRaw, chExtractorErrors, chJobErrors, &wgActiveBatches)
 		})
 	}
 
-	wgActivePartitions.Add(len(partitions))
+	wgActiveBatches.Add(len(batches))
 	go func() {
-		for _, batch := range partitions {
-			chPartitions <- batch
+		for _, batch := range batches {
+			chBatches <- batch
 		}
 	}()
 
-	log.Infof("Starting %d transformer(s)...", maxExtractors)
+	log.Infof("Starting %d transformers...", maxExtractors)
+	transformStartTime := time.Now()
 
 	for range maxExtractors {
 		wgTransformers.Go(func() {
-			transformer.Exec(
-				localCtx,
-				sourceColTypes,
-				chBatchesRaw,
-				chBatchesTransformed,
-				chJobErrors,
-				&wgActiveBatches,
-			)
+			transformRowsMssql(jobCtx, sourceColTypes, chChunksRaw, chChunksTransformed, chJobErrors, &wgActiveChunks)
 		})
 	}
 
-	log.Infof("Starting %d loader(s)...", job.MaxLoaders)
+	log.Infof("Starting %d PostgreSQL loader(s)...", NumLoaders)
+	loadStartTime := time.Now()
 
-	for range job.MaxLoaders {
+	for range NumLoaders {
 		wgLoaders.Go(func() {
-			loader.Exec(
-				localCtx,
-				job.TargetTable,
-				targetColTypes,
-				chBatchesTransformed,
-				chLoadersErrors,
-				chJobErrors,
-				&wgActiveBatches,
-				&rowsLoaded,
-			)
+			loadRowsPostgres(jobCtx, targetDb, job, targetColTypes, chChunksTransformed, chLoadersErrors, chJobErrors, &wgActiveChunks)
 		})
 	}
 
 	go func() {
-		log.Debugf("Waiting for goroutines (%v)", job.Name)
-
-		wgActivePartitions.Wait()
-		log.Debugf("wgActivePartitions is empty (%v)", job.Name)
-		close(chPartitions)
-		log.Debugf("chPartitions is closed (%v)", job.Name)
+		wgActiveBatches.Wait()
+		close(chBatches)
+		close(chExtractorErrors)
 
 		wgExtractors.Wait()
-		log.Debugf("wgExtractors is empty (%v)", job.Name)
-		close(chBatchesRaw)
-		log.Debugf("chBatchesRaw is closed (%v)", job.Name)
+		log.Infof("Extraction completed in %v", time.Since(extractStartTime))
+		close(chChunksRaw)
 
 		wgTransformers.Wait()
-		log.Debugf("wgTransformers is empty (%v)", job.Name)
+		log.Infof("Transformation completed in %v", time.Since(transformStartTime))
 
-		wgActiveBatches.Wait()
-		log.Debugf("wgActiveBatches is empty (%v)", job.Name)
-		close(chBatchesTransformed)
-		log.Debugf("chBatchesTransformed is empty (%v)", job.Name)
+		wgActiveChunks.Wait()
+		close(chChunksTransformed)
 		close(chLoadersErrors)
-		log.Debugf("chLoadersErrors is empty (%v)", job.Name)
 
 		wgLoaders.Wait()
-		log.Debugf("wgLoaders is empty (%v)", job.Name)
+		log.Infof("Loading completed in %v", time.Since(loadStartTime))
 
 		cancel()
 	}()
 
-	for _, query := range job.PostSQL {
-		if _, err := targetDbWrapper.Exec(localCtx, query); err != nil {
-			result.Error = err
-			return result
-		}
-	}
-
-	log.Debugf("waiting for local context to be done (%v)", job.Name)
-	<-localCtx.Done()
-	log.Debugf("local context done (%v)", job.Name)
-
-	if ctx.Err() != nil {
-		result.Error = ctx.Err()
-	}
-
-	result.Duration = time.Since(result.StartTime)
-	result.RowsRead = atomic.LoadInt64(&rowsRead)
-	result.RowsLoaded = atomic.LoadInt64(&rowsLoaded)
-	result.RowsFailed = atomic.LoadInt64(&rowsFailed)
-
-	if result.RowsRead != result.RowsLoaded {
-		result.Error = fmt.Errorf("Row count mismatch: extracted %d rows but loaded %d rows (failed: %d)", result.RowsRead, result.RowsLoaded, result.RowsFailed)
-	}
-
-	return result
+	<-jobCtx.Done()
+	log.Infof("Migration job completed. Total time: %v", time.Since(jobStartTime))
+}
+
+func logColumnTypes(columnTypes []ColumnType, label string) {
+	log.Debug(label)
+
+	for _, col := range columnTypes {
+		log.Debugf("%+v", col)
+	}
+}
+
+func logSampleRow(job MigrationJob, columns []ColumnType, rowValues UnknownRowValues, tag string) {
+	log.Infof("[%s.%s] Sample row: (%s)", job.Schema, job.Table, tag)
+	for i, col := range columns {
+		log.Infof("%s (%T): %v", col.Name(), rowValues[i], rowValues[i])
+	}
 }

cmd/go_migrate/transformer.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type transformerFunc func(any) (any, error)
+
+type columnTransformPlan struct {
+	index int
+	fn    transformerFunc
+}
+
+func transformRowsMssql(
+	ctx context.Context,
+	columns []ColumnType,
+	chChunksIn <-chan Chunk,
+	chChunksOut chan<- Chunk,
+	chJobErrorsOut chan<- JobError,
+	wgActiveChunks *sync.WaitGroup,
+) {
+	transformationPlan := computeTransformationPlan(columns)
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+
+		case chunk, ok := <-chChunksIn:
+			if !ok {
+				return
+			}
+
+			if len(transformationPlan) == 0 {
+				select {
+				case chChunksOut <- chunk:
+					wgActiveChunks.Add(1)
+					continue
+				case <-ctx.Done():
+					return
+				}
+			}
+
+			chunkStartTime := time.Now()
+
+			err := processChunk(ctx, &chunk, transformationPlan)
+			if err != nil {
+				if errors.Is(err, ctx.Err()) {
+					return
+				}
+
+				select {
+				case chJobErrorsOut <- JobError{ShouldCancelJob: true, Msg: "Transformation failed", Prev: err}:
+				case <-ctx.Done():
+				}
+				return
+			}
+
+			log.Infof("Transformed chunk %s: %d rows in %v", chunk.Id, len(chunk.Data), time.Since(chunkStartTime))
+
+			select {
+			case chChunksOut <- chunk:
+			case <-ctx.Done():
+				return
+			}
+
+			wgActiveChunks.Add(1)
+		}
+	}
+}
+
+func computeTransformationPlan(columns []ColumnType) []columnTransformPlan {
+	var plan []columnTransformPlan
+
+	for i, col := range columns {
+		switch col.SystemType() {
+		case "uniqueidentifier":
+			plan = append(plan, columnTransformPlan{
+				index: i,
+				fn: func(v any) (any, error) {
+					if b, ok := v.([]byte); ok && b != nil {
+						return mssqlUuidToBigEndian(b)
+					}
+					return v, nil
+				},
+			})
+
+		case "geometry", "geography":
+			plan = append(plan, columnTransformPlan{
+				index: i,
+				fn: func(v any) (any, error) {
+					if b, ok := v.([]byte); ok && b != nil {
+						return wkbToEwkbWithSrid(b, 4326)
+					}
+					return v, nil
+				},
+			})
+
+		case "datetime", "datetime2":
+			plan = append(plan, columnTransformPlan{
+				index: i,
+				fn: func(v any) (any, error) {
+					if t, ok := v.(time.Time); ok {
+						return ensureUTC(t), nil
+					}
+					return v, nil
+				},
+			})
+		}
+	}
+
+	return plan
+}
+
+const processChunkCtxCheck = 4096
+
+func processChunk(ctx context.Context, chunk *Chunk, transformationPlan []columnTransformPlan) error {
+	for i, rowValues := range chunk.Data {
+		if i%processChunkCtxCheck == 0 {
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+		}
+
+		for _, task := range transformationPlan {
+			val := rowValues[task.index]
+			if val == nil {
+				continue
+			}
+
+			transformed, err := task.fn(val)
+			if err != nil {
+				return err
+			}
+			rowValues[task.index] = transformed
+		}
+	}
+
+	return nil
+}

config.yaml

@@ -1,22 +1,15 @@
-max_parallel_workers: 4
-source_db_type: sqlserver
-target_db_type: postgres
+max_parallel_workers: 2
 
 defaults:
-  max_extractors: 2
-  max_loaders: 4
+  max_extractors: 4
+  max_loaders: 8
   queue_size: 8
-  batch_size: 25000
-  batches_per_partition: 8
+  chunk_size: 50000
+  chunks_per_batch: 10
   truncate_target: true
   truncate_method: TRUNCATE # TRUNCATE | DELETE
-  max_partition_errrors: 5
-  max_chunk_errors: 5
   retry:
     attempts: 3
-    base_delay_ms: 500
-    max_delay_ms: 10000
-    max_jitter_ms: 500
 
 jobs:
   - name: cartografia_manzana
@@ -28,14 +21,19 @@ jobs:
     target:
       schema: Cartografia
       table: MANZANA
+    max_extractors: 2 # overrides default config
+    max_loaders: 4 # overrides default config
+    queue_size: 4 # overrides default config
+    chunk_size: 25000 # overrides default config
+    chunks_per_batch: 8 # overrides default config
+    truncate_target: false # overrides default config
+    truncate_method: DELETE # overrides default config
+    retry:
+      attempts: 5 # overrides default config
     pre_sql:
-      - 'SELECT 1'
-      # - 'TRUNCATE TABLE "Cartografia"."MANZANA"'
-    range:
-      min: 1000000
-      max: 2000000
-      is_min_inclusive: false
-      is_max_inclusive: true
+      - "SELECT 1"
+    post_sql:
+      - "SELECT 2"
 
   - name: red_puerto
     enabled: true
@@ -46,8 +44,3 @@ jobs:
     target:
       schema: Red
       table: PUERTO
-    pre_sql: 
-      - 'SELECT 1'
-      # - 'TRUNCATE TABLE "Red"."PUERTO"'
-    post_sql: 
-      - "SELECT 1"

go.mod

@@ -4,14 +4,13 @@ go 1.25.7
 
 require (
 	github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4
+	github.com/goccy/go-yaml v1.19.2
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.9.1
 	github.com/joho/godotenv v1.5.1
 	github.com/microsoft/go-mssqldb v1.9.8
 	github.com/sirupsen/logrus v1.9.4
 	github.com/twpayne/go-geom v1.6.1
-	golang.org/x/sync v0.19.0
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -20,10 +19,9 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
 )

go.sum

@@ -16,12 +16,13 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ
 github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
 github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4 h1:4vH4+3zfwZTqoJEFw7DsTaH1V8jgVwnyeDvNi2TxzAc=
 github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4/go.mod h1:jlB0I5BIfcJBGdV6rRGPthSBfeY86RGkSAwcsldbHJc=
+github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
+github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
 github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
@@ -42,10 +43,6 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
-github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/microsoft/go-mssqldb v1.9.8 h1:d4IFMvF/o+HdpXUqbBfzHvn/NlFA75YGcfHUUvDFJEM=
@@ -54,8 +51,6 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
-github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
 github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
@@ -78,8 +73,6 @@ golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
 golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/app/config/main.go

@@ -8,8 +8,10 @@ import (
 )
 
 type appConfig struct {
-	SourceDbUrl string
-	TargetDbUrl string
+	SourceDbUrl  string
+	SourceDbType string
+	TargetDbUrl  string
+	TargetDbType string
 }
 
 func loadEnv() {
@@ -27,14 +29,26 @@ func getAppConfig() appConfig {
 		log.Fatal("SOURCE_DB_URL environment variable not set")
 	}
 
+	sourceDbType := os.Getenv("SOURCE_DB_TYPE")
+	if sourceDbType == "" {
+		log.Fatal("SOURCE_DB_TYPE environment variable not set")
+	}
+
 	targetDbUrl := os.Getenv("TARGET_DB_URL")
 	if targetDbUrl == "" {
 		log.Fatal("TARGET_DB_URL environment variable not set")
 	}
 
+	targetDbType := os.Getenv("TARGET_DB_TYPE")
+	if targetDbType == "" {
+		log.Fatal("TARGET_DB_TYPE environment variable not set")
+	}
+
 	return appConfig{
-		SourceDbUrl: sourceDbUrl,
-		TargetDbUrl: targetDbUrl,
+		SourceDbUrl:  sourceDbUrl,
+		SourceDbType: sourceDbType,
+		TargetDbUrl:  targetDbUrl,
+		TargetDbType: targetDbType,
 	}
 }
 

internal/app/config/migration.go

@@ -1,138 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"os"
-
-	"gopkg.in/yaml.v3"
-)
-
-type RetryConfig struct {
-	Attempts    int `yaml:"attempts"`
-	BaseDelayMs int `yaml:"base_delay_ms"`
-	MaxDelayMs  int `yaml:"max_delay_ms"`
-	MaxJitterMs int `yaml:"max_jitter_ms"`
-}
-
-type JobConfig struct {
-	MaxExtractors       int         `yaml:"max_extractors"`
-	MaxLoaders          int         `yaml:"max_loaders"`
-	QueueSize           int         `yaml:"queue_size"`
-	BatchSize           int         `yaml:"batch_size"`
-	BatchesPerPartition int         `yaml:"batches_per_partition"`
-	TruncateTarget      bool        `yaml:"truncate_target"`
-	TruncateMethod      string      `yaml:"truncate_method"`
-	MaxPartitionErrrors int         `yaml:"max_partition_errrors"`
-	MaxChunkErrors      int         `yaml:"max_chunk_errors"`
-	Retry               RetryConfig `yaml:"retry"`
-	RowsPerPartition    int64
-}
-
-type TableInfo struct {
-	Schema string `yaml:"schema"`
-	Table  string `yaml:"table"`
-}
-
-type TargetTableInfo struct {
-	TableInfo `yaml:",inline"`
-}
-
-type SourceTableInfo struct {
-	TableInfo  `yaml:",inline"`
-	PrimaryKey string `yaml:"primary_key"`
-}
-
-type Job struct {
-	Name        string          `yaml:"name"`
-	Enabled     bool            `yaml:"enabled"`
-	SourceTable SourceTableInfo `yaml:"source"`
-	TargetTable TargetTableInfo `yaml:"target"`
-	PreSQL      []string        `yaml:"pre_sql"`
-	PostSQL     []string        `yaml:"post_sql"`
-	JobConfig   `yaml:",inline"`
-	Range       struct {
-		Min            int64 `yaml:"min"`
-		Max            int64 `yaml:"max"`
-		IsMinInclusive bool  `yaml:"is_min_inclusive"`
-		IsMaxInclusive bool  `yaml:"is_max_inclusive"`
-	}
-}
-
-type MigrationConfig struct {
-	MaxParallelWorkers int       `yaml:"max_parallel_workers"`
-	SourceDbType       string    `yaml:"source_db_type"`
-	TargetDbType       string    `yaml:"target_db_type"`
-	Defaults           JobConfig `yaml:"defaults"`
-	Jobs               []Job     `yaml:"jobs"`
-}
-
-type rawConfig struct {
-	MaxParallelWorkers int         `yaml:"max_parallel_workers"`
-	SourceDbType       string      `yaml:"source_db_type"`
-	TargetDbType       string      `yaml:"target_db_type"`
-	Defaults           JobConfig   `yaml:"defaults"`
-	Jobs               []yaml.Node `yaml:"jobs"`
-}
-
-func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
-	var raw rawConfig
-	if err := value.Decode(&raw); err != nil {
-		return err
-	}
-
-	c.MaxParallelWorkers = raw.MaxParallelWorkers
-	c.Defaults = raw.Defaults
-	c.SourceDbType = raw.SourceDbType
-	c.TargetDbType = raw.TargetDbType
-	c.Defaults.RowsPerPartition = int64(raw.Defaults.BatchSize * raw.Defaults.BatchesPerPartition)
-
-	for _, node := range raw.Jobs {
-		job := Job{
-			JobConfig: raw.Defaults,
-		}
-
-		if err := node.Decode(&job); err != nil {
-			return err
-		}
-
-		job.RowsPerPartition = int64(job.BatchSize * job.BatchesPerPartition)
-
-		c.Jobs = append(c.Jobs, job)
-	}
-
-	return nil
-}
-
-const defaultConfigFileName string = "config.yaml"
-
-func filenamesOrDefault(filenames []string) []string {
-	if len(filenames) == 0 {
-		return []string{defaultConfigFileName}
-	}
-	return filenames
-}
-
-func ReadMigrationConfig(filenames ...string) (MigrationConfig, error) {
-	filenames = filenamesOrDefault(filenames)
-	var data []byte
-	var err error
-
-	for _, filename := range filenames {
-		data, err = os.ReadFile(filename)
-		if err != nil {
-			continue
-		}
-		break
-	}
-
-	if err != nil {
-		return MigrationConfig{}, fmt.Errorf("Error reading config file: %v", err)
-	}
-
-	var config MigrationConfig
-	if err := yaml.Unmarshal(data, &config); err != nil {
-		return MigrationConfig{}, fmt.Errorf("Error parsing config file: %v", err)
-	}
-
-	return config, nil
-}

internal/app/convert/main.go

@@ -1,18 +0,0 @@
-package convert
-
-func ToInt64(v any) (int64, bool) {
-	switch t := v.(type) {
-	case int:
-		return int64(t), true
-	case int8:
-		return int64(t), true
-	case int16:
-		return int64(t), true
-	case int32:
-		return int64(t), true
-	case int64:
-		return int64(t), true
-	default:
-		return 0, false
-	}
-}

internal/app/custom_errors/backoff.go

@@ -1,61 +0,0 @@
-package custom_errors
-
-import (
-	"context"
-	"math/rand"
-	"time"
-)
-
-func computeBackoffDelay(retryCounter int, baseDelayMs int, maxDelayMs int, maxJitterMs int) time.Duration {
-	if retryCounter < 0 {
-		retryCounter = 0
-	}
-
-	delay := max(time.Duration(baseDelayMs)*time.Millisecond, 0)
-
-	maxDelay := time.Duration(maxDelayMs) * time.Millisecond
-	for i := 0; i < retryCounter; i++ {
-		if maxDelayMs > 0 && delay >= maxDelay {
-			delay = maxDelay
-			break
-		}
-		if delay == 0 {
-			break
-		}
-		delay *= 2
-	}
-
-	if maxDelayMs > 0 && delay > maxDelay {
-		delay = maxDelay
-	}
-
-	if maxJitterMs > 0 {
-		jitter := time.Duration(rand.Intn(maxJitterMs+1)) * time.Millisecond
-		delay += jitter
-	}
-
-	if delay < 0 {
-		delay = 0
-	}
-
-	return delay
-}
-
-func requeueWithBackoff(ctx context.Context, delay time.Duration, enqueue func()) {
-	if delay <= 0 {
-		enqueue()
-		return
-	}
-
-	go func() {
-		timer := time.NewTimer(delay)
-		defer timer.Stop()
-
-		select {
-		case <-ctx.Done():
-			return
-		case <-timer.C:
-			enqueue()
-		}
-	}()
-}

internal/app/custom_errors/extractor.error.go

@@ -1,16 +0,0 @@
-package custom_errors
-
-import (
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-type ExtractorError struct {
-	Partition models.Partition
-	LastId    int64
-	HasLastId bool
-	Msg       string
-}
-
-func (e *ExtractorError) Error() string {
-	return e.Msg
-}

internal/app/custom_errors/loader.error.go

@@ -1,107 +0,0 @@
-package custom_errors
-
-import (
-	"context"
-	"fmt"
-	"sync"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-type LoaderError struct {
-	Batch models.Batch
-	Msg   string
-}
-
-func (e *LoaderError) Error() string {
-	return e.Msg
-}
-
-func LoaderErrorHandler(
-	ctx context.Context,
-	retryConfig config.RetryConfig,
-	maxChunkErrors int,
-	chErrorsIn <-chan LoaderError,
-	chBatchesOut chan<- models.Batch,
-	chJobErrorsOut chan<- JobError,
-	wgActiveBatches *sync.WaitGroup,
-) {
-	definitiveErrors := 0
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-
-		case err, ok := <-chErrorsIn:
-			if !ok {
-				return
-			}
-
-			if err.Batch.RetryCounter >= retryConfig.Attempts {
-				wgActiveBatches.Done()
-				definitiveErrors++
-				jobError := JobError{
-					ShouldCancelJob: false,
-					Msg:             fmt.Sprintf("Batch %v reached max retries (%d)", err.Batch.Id, retryConfig.Attempts),
-					Prev:            &err,
-				}
-
-				select {
-				case chJobErrorsOut <- jobError:
-				case <-ctx.Done():
-					return
-				}
-
-				if maxChunkErrors > 0 && definitiveErrors >= maxChunkErrors {
-					fatalError := JobError{
-						ShouldCancelJob: true,
-						Msg:             fmt.Sprintf("Chunk error limit reached (%d)", maxChunkErrors),
-						Prev:            &err,
-					}
-
-					select {
-					case chJobErrorsOut <- fatalError:
-					case <-ctx.Done():
-						return
-					}
-				}
-
-				continue
-			} else {
-				jobError := JobError{
-					ShouldCancelJob: false,
-					Msg:             fmt.Sprintf("Temporal error in batch %v (retries: %d)", err.Batch.Id, err.Batch.RetryCounter),
-					Prev:            &err,
-				}
-
-				select {
-				case chJobErrorsOut <- jobError:
-				case <-ctx.Done():
-					return
-				}
-			}
-
-			err.Batch.RetryCounter++
-			delay := computeBackoffDelay(
-				err.Batch.RetryCounter,
-				retryConfig.BaseDelayMs,
-				retryConfig.MaxDelayMs,
-				retryConfig.MaxJitterMs,
-			)
-
-			requeueWithBackoff(ctx, delay, func() {
-				select {
-				case chBatchesOut <- err.Batch:
-				case <-ctx.Done():
-					return
-				}
-			})
-		}
-	}
-}

internal/app/db-wrapper/main.go

@@ -1,19 +0,0 @@
-package dbwrapper
-
-import "fmt"
-
-type Factory func() DbWrapper
-
-var drivers = make(map[string]Factory)
-
-func Register(name string, factory Factory) {
-	drivers[name] = factory
-}
-
-func New(driverType string) (DbWrapper, error) {
-	factory, ok := drivers[driverType]
-	if !ok {
-		return nil, fmt.Errorf("driver not yet supported: %s", driverType)
-	}
-	return factory(), nil
-}

internal/app/db-wrapper/mssql.go

@@ -1,176 +0,0 @@
-package dbwrapper
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-
-	mssql "github.com/microsoft/go-mssqldb"
-)
-
-func init() {
-	Register("sqlserver", func() DbWrapper {
-		return &mssqlDbWrapper{dialect: "sqlserver"}
-	})
-}
-
-type mssqlRowResult struct {
-	row *sql.Row
-}
-
-func (mr *mssqlRowResult) Scan(dest ...any) error {
-	return mr.row.Scan(dest...)
-}
-
-type mssqlRowsResult struct {
-	columns []string
-	rows    *sql.Rows
-}
-
-func (mr *mssqlRowsResult) Close() error {
-	return mr.rows.Close()
-}
-
-func (mr *mssqlRowsResult) Columns() ([]string, error) {
-	if mr.columns != nil {
-		return mr.columns, nil
-	}
-
-	return mr.rows.Columns()
-}
-
-func (mr *mssqlRowsResult) Err() error {
-	return mr.rows.Err()
-}
-
-func (mr *mssqlRowsResult) Next() bool {
-	return mr.rows.Next()
-}
-
-func (mr *mssqlRowsResult) Scan(dest ...any) error {
-	return mr.rows.Scan(dest...)
-}
-
-func (mr *mssqlRowsResult) Values() ([]any, error) {
-	columns, err := mr.Columns()
-	if err != nil {
-		return nil, err
-	}
-
-	rowValues := make([]any, len(columns))
-	scanArgs := make([]any, len(columns))
-	for i := range rowValues {
-		scanArgs[i] = &rowValues[i]
-	}
-
-	if err := mr.rows.Scan(scanArgs...); err != nil {
-		return nil, err
-	}
-
-	return rowValues, nil
-}
-
-type mssqlDbWrapper struct {
-	db      *sql.DB
-	dialect string
-}
-
-func (mw *mssqlDbWrapper) Connect(ctx context.Context, dbUrl string) error {
-	db, err := sql.Open("sqlserver", dbUrl)
-	if err != nil {
-		return err
-	}
-
-	if err := db.PingContext(ctx); err != nil {
-		if err := db.Close(); err != nil {
-			return err
-		}
-		return err
-	}
-
-	mw.db = db
-	return nil
-}
-
-func (mw *mssqlDbWrapper) Close() error {
-	return mw.db.Close()
-}
-
-func (mw *mssqlDbWrapper) Exec(ctx context.Context, query string, args ...any) (ExecResult, error) {
-	result, execErr := mw.db.ExecContext(ctx, query, args...)
-	if execErr != nil {
-		return ExecResult{}, execErr
-	}
-
-	affectedRows, err := result.RowsAffected()
-	if err != nil {
-		return ExecResult{}, err
-	}
-
-	return ExecResult{AffectedRows: affectedRows}, nil
-}
-
-func (mw *mssqlDbWrapper) GetDialect() string {
-	return mw.dialect
-}
-
-func (mw *mssqlDbWrapper) Query(ctx context.Context, query string, args ...any) (RowsResult, error) {
-	rows, err := mw.db.QueryContext(ctx, query, args...)
-	if err != nil {
-		return nil, err
-	}
-
-	return &mssqlRowsResult{columns: nil, rows: rows}, nil
-}
-
-func (mw *mssqlDbWrapper) QueryRow(ctx context.Context, query string, args ...any) RowResult {
-	row := mw.db.QueryRowContext(ctx, query, args...)
-	return &mssqlRowResult{row: row}
-}
-
-func (mw *mssqlDbWrapper) SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error) {
-	tx, err := mw.db.BeginTx(ctx, nil)
-	if err != nil {
-		return 0, err
-	}
-
-	fullTableName := fmt.Sprintf("[%s].[%s]", schema, table)
-
-	stmt, err := tx.PrepareContext(ctx, mssql.CopyIn(fullTableName, mssql.BulkOptions{}, columnNames...))
-	if err != nil {
-		tx.Rollback()
-		return 0, err
-	}
-
-	for _, row := range rows {
-		_, err = stmt.ExecContext(ctx, row...)
-		if err != nil {
-			stmt.Close()
-			tx.Rollback()
-			return 0, err
-		}
-	}
-
-	result, err := stmt.ExecContext(ctx)
-	if err != nil {
-		stmt.Close()
-		tx.Rollback()
-		return 0, err
-	}
-
-	if err := stmt.Close(); err != nil {
-		tx.Rollback()
-		return 0, err
-	}
-
-	if err := tx.Commit(); err != nil {
-		return 0, err
-	}
-
-	rowsAffected, raErr := result.RowsAffected()
-	if raErr != nil {
-		return 0, nil
-	}
-
-	return rowsAffected, nil
-}

internal/app/db-wrapper/postgres.go

@@ -1,128 +0,0 @@
-package dbwrapper
-
-import (
-	"context"
-	"errors"
-
-	"github.com/jackc/pgx/v5"
-	"github.com/jackc/pgx/v5/pgxpool"
-)
-
-func init() {
-	Register("postgres", func() DbWrapper {
-		return &postgresDbWrapper{dialect: "postgres"}
-	})
-}
-
-type postgresRowResult struct {
-	row pgx.Row
-}
-
-func (pr *postgresRowResult) Scan(dest ...any) error {
-	return pr.row.Scan(dest...)
-}
-
-type postgresRowsResult struct {
-	columns []string
-	rows    pgx.Rows
-}
-
-func (pr *postgresRowsResult) Close() error {
-	pr.rows.Close()
-	return nil
-}
-
-func (pr *postgresRowsResult) Columns() ([]string, error) {
-	if pr.columns != nil {
-		return pr.columns, nil
-	}
-
-	rawColumns := pr.rows.FieldDescriptions()
-	if rawColumns == nil {
-		return nil, errors.New("error retrieving columns")
-	}
-
-	columns := make([]string, 0, len(rawColumns))
-	for _, rc := range rawColumns {
-		columns = append(columns, rc.Name)
-	}
-
-	return columns, nil
-}
-
-func (pr *postgresRowsResult) Err() error {
-	return pr.rows.Err()
-}
-
-func (pr *postgresRowsResult) Next() bool {
-	return pr.rows.Next()
-}
-
-func (pr *postgresRowsResult) Scan(dest ...any) error {
-	return pr.rows.Scan(dest...)
-}
-
-func (pr *postgresRowsResult) Values() ([]any, error) {
-	return pr.rows.Values()
-}
-
-type postgresDbWrapper struct {
-	db      *pgxpool.Pool
-	dialect string
-}
-
-func (pw *postgresDbWrapper) Connect(ctx context.Context, dbUrl string) error {
-	pool, err := pgxpool.New(ctx, dbUrl)
-	if err != nil {
-		return err
-	}
-
-	if err := pool.Ping(ctx); err != nil {
-		pool.Close()
-		return err
-	}
-
-	pw.db = pool
-	return nil
-}
-
-func (pw *postgresDbWrapper) Close() error {
-	pw.db.Close()
-	return nil
-}
-
-func (pw *postgresDbWrapper) Exec(ctx context.Context, query string, args ...any) (ExecResult, error) {
-	result, err := pw.db.Exec(ctx, query, args...)
-	if err != nil {
-		return ExecResult{}, err
-	}
-
-	return ExecResult{AffectedRows: result.RowsAffected()}, nil
-}
-
-func (pw *postgresDbWrapper) GetDialect() string {
-	return pw.dialect
-}
-
-func (pw *postgresDbWrapper) Query(ctx context.Context, query string, args ...any) (RowsResult, error) {
-	rows, err := pw.db.Query(ctx, query, args...)
-	if err != nil {
-		return nil, err
-	}
-
-	return &postgresRowsResult{columns: nil, rows: rows}, nil
-}
-
-func (pw *postgresDbWrapper) QueryRow(ctx context.Context, query string, args ...any) RowResult {
-	row := pw.db.QueryRow(ctx, query, args...)
-	return &postgresRowResult{row: row}
-}
-
-func (pw *postgresDbWrapper) SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error) {
-	affectedRows, err := pw.db.CopyFrom(ctx, pgx.Identifier{schema, table}, columnNames, pgx.CopyFromRows(rows))
-	if err != nil {
-		return 0, err
-	}
-
-	return affectedRows, nil
-}

internal/app/db-wrapper/types.go

@@ -1,35 +0,0 @@
-package dbwrapper
-
-import (
-	"context"
-	"errors"
-)
-
-var MethodNotSupported error = errors.New("Method not supported by driver... yet :P")
-
-type ExecResult struct {
-	AffectedRows int64
-}
-
-type RowsResult interface {
-	Close() error
-	Columns() ([]string, error)
-	Err() error
-	Next() bool
-	Scan(dest ...any) error
-	Values() ([]any, error)
-}
-
-type RowResult interface {
-	Scan(dest ...any) error
-}
-
-type DbWrapper interface {
-	Close() error
-	Connect(ctx context.Context, dbUrl string) error
-	Exec(ctx context.Context, query string, args ...any) (ExecResult, error)
-	GetDialect() string
-	Query(ctx context.Context, query string, args ...any) (RowsResult, error)
-	QueryRow(ctx context.Context, query string, args ...any) RowResult
-	SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error)
-}

internal/app/db/postgres.go

@@ -0,0 +1,28 @@
+package db
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+func Connect(ctx context.Context, dbURL string) (*pgxpool.Pool, error) {
+	pool, err := pgxpool.New(ctx, dbURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to connect to database: %w", err)
+	}
+
+	if err := pool.Ping(ctx); err != nil {
+		pool.Close()
+		return nil, fmt.Errorf("unable to ping database: %w", err)
+	}
+
+	return pool, nil
+}
+
+func Close(pool *pgxpool.Pool) {
+	if pool != nil {
+		pool.Close()
+	}
+}

internal/app/etl/extractors/consumer.go

@@ -1,92 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"errors"
-	"slices"
-	"strings"
-	"sync"
-	"sync/atomic"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-func Consume(
-	ctx context.Context,
-	extractor etl.Extractor,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	chPartitionsIn <-chan models.Partition,
-	chBatchesOut chan<- models.Batch,
-	chErrorsOut chan<- custom_errors.JobError,
-	wgActivePartitions *sync.WaitGroup,
-	rowsRead *int64,
-) {
-	indexPrimaryKey := slices.IndexFunc(columns, func(col models.ColumnType) bool {
-		return strings.EqualFold(col.Name(), tableInfo.PrimaryKey)
-	})
-
-	if indexPrimaryKey == -1 {
-		select {
-		case <-ctx.Done():
-			return
-		case chErrorsOut <- custom_errors.JobError{
-			ShouldCancelJob: true,
-			Msg:             "Primary key not found in provided columns",
-		}:
-		}
-		return
-	}
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		case partition, ok := <-chPartitionsIn:
-			if !ok {
-				return
-			}
-
-			rowsReadResult, err := extractWithRetries(
-				ctx,
-				extractor,
-				tableInfo,
-				columns,
-				batchSize,
-				partition,
-				indexPrimaryKey,
-				chBatchesOut,
-			)
-			wgActivePartitions.Done()
-
-			if rowsReadResult > 0 {
-				atomic.AddInt64(rowsRead, rowsReadResult)
-			}
-
-			if err != nil {
-				var jobError *custom_errors.JobError
-				if errors.As(err, &jobError) {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- *jobError:
-					}
-				} else {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: false, Msg: err.Error(), Prev: err}:
-					}
-				}
-			}
-		}
-	}
-}

internal/app/etl/extractors/extract-with-retries.go

@@ -1,70 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-func extractWithRetries(
-	ctx context.Context,
-	extractor etl.Extractor,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	partition models.Partition,
-	indexPrimaryKey int,
-	chBatchesOut chan<- models.Batch,
-) (int64, error) {
-	var totalRowsRead int64
-	delay := time.Duration(time.Second * 1)
-	currentParitition := partition
-
-	for {
-		rowsRead, err := extractor.Exec(
-			ctx,
-			tableInfo,
-			columns,
-			batchSize,
-			currentParitition,
-			indexPrimaryKey,
-			chBatchesOut,
-		)
-		totalRowsRead += rowsRead
-
-		if err == nil {
-			return totalRowsRead, nil
-		}
-
-		var exError *custom_errors.ExtractorError
-		if errors.As(err, &exError) {
-			currentParitition.RetryCounter++
-
-			if currentParitition.RetryCounter > 3 {
-				return totalRowsRead, &custom_errors.JobError{
-					Msg:  fmt.Sprintf("Partition %v reached max retries", exError.Partition.Id),
-					Prev: err,
-				}
-			}
-
-			if exError.HasLastId {
-				currentParitition.ParentId = exError.Partition.Id
-				currentParitition.Id = uuid.New()
-				currentParitition.Range.Min = exError.LastId
-				currentParitition.Range.IsMinInclusive = false
-			}
-
-			time.Sleep(delay)
-			continue
-		}
-
-		return totalRowsRead, err
-	}
-}

internal/app/etl/extractors/main.go

@@ -1,64 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"fmt"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/convert"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-func errorFromLastPartitionRow(
-	lastRow models.UnknownRowValues,
-	indexPrimaryKey int,
-	partition models.Partition,
-	previousError error,
-) error {
-	lastIdRawValue := lastRow[indexPrimaryKey]
-
-	lastId, ok := convert.ToInt64(lastIdRawValue)
-	if !ok {
-		currentPartition := partition
-		currentPartition.RetryCounter = 3
-		return &custom_errors.ExtractorError{
-			Partition: currentPartition,
-			HasLastId: true,
-			Msg:       fmt.Sprintf("Couldn't cast last id value as int: %s", previousError.Error()),
-		}
-
-	}
-
-	return &custom_errors.ExtractorError{
-		Partition: partition,
-		HasLastId: true,
-		LastId:    lastId,
-		Msg:       previousError.Error(),
-	}
-}
-
-func sendBatch(ctx context.Context, chBatchesOut chan<- models.Batch, batch models.Batch) error {
-	select {
-	case chBatchesOut <- batch:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}
-
-func flush(
-	ctx context.Context,
-	partition *models.Partition,
-	batchSize int,
-	batchRows []models.UnknownRowValues,
-	chBatchesOut chan<- models.Batch,
-) error {
-	if len(batchRows) == 0 {
-		return nil
-	}
-
-	batch := models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows}
-	batchRows = make([]models.UnknownRowValues, 0, batchSize)
-	return sendBatch(ctx, chBatchesOut, batch)
-}

internal/app/etl/extractors/mssql.go

@@ -1,121 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"strings"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-type MssqlExtractor struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewMssqlExtractor(db dbwrapper.DbWrapper) etl.Extractor {
-	return &MssqlExtractor{db: db}
-}
-
-func buildExtractQueryMssql(
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	includeRange bool,
-	isMinInclusive bool,
-) string {
-	var sbQuery strings.Builder
-
-	sbQuery.WriteString("SELECT ")
-
-	if len(columns) == 0 {
-		sbQuery.WriteString("*")
-	} else {
-		for i, col := range columns {
-			fmt.Fprintf(&sbQuery, "[%s]", col.Name())
-
-			if col.Type() == "GEOMETRY" {
-				fmt.Fprintf(&sbQuery, ".STAsBinary() AS [%s]", col.Name())
-			}
-
-			if i < len(columns)-1 {
-				sbQuery.WriteString(", ")
-			}
-		}
-	}
-
-	fmt.Fprintf(&sbQuery, " FROM [%s].[%s]", tableInfo.Schema, tableInfo.Table)
-
-	if includeRange {
-		fmt.Fprintf(&sbQuery, " WHERE [%s]", tableInfo.PrimaryKey)
-		if isMinInclusive {
-			sbQuery.WriteString(" >=")
-		} else {
-			sbQuery.WriteString(" >")
-		}
-
-		fmt.Fprintf(&sbQuery, " @min AND [%s] <= @max", tableInfo.PrimaryKey)
-	}
-
-	fmt.Fprintf(&sbQuery, " ORDER BY [%s] ASC", tableInfo.PrimaryKey)
-
-	return sbQuery.String()
-}
-
-func (mssqlEx *MssqlExtractor) Exec(
-	ctx context.Context,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	partition models.Partition,
-	indexPrimaryKey int,
-	chBatchesOut chan<- models.Batch,
-) (int64, error) {
-	query := buildExtractQueryMssql(tableInfo, columns, partition.HasRange, partition.Range.IsMinInclusive)
-
-	var queryArgs []any
-	if partition.HasRange {
-		queryArgs = append(queryArgs, sql.Named("min", partition.Range.Min), sql.Named("max", partition.Range.Max))
-	}
-
-	rows, err := mssqlEx.db.Query(ctx, query, queryArgs...)
-	if err != nil {
-		return 0, err
-	}
-	defer rows.Close()
-
-	batchRows := make([]models.UnknownRowValues, 0, batchSize)
-	var rowsRead int64 = 0
-
-	for rows.Next() {
-		values, err := rows.Values()
-		if err != nil {
-			if len(batchRows) == 0 {
-				return rowsRead, err
-			}
-
-			if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
-				return rowsRead, err
-			}
-
-			lastRow := batchRows[len(batchRows)-1]
-			return rowsRead, errorFromLastPartitionRow(lastRow, indexPrimaryKey, partition, err)
-		}
-		rowsRead++
-
-		batchRows = append(batchRows, values)
-		if len(batchRows) >= batchSize {
-			if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
-				return rowsRead, err
-			}
-		}
-	}
-
-	if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
-		return rowsRead, err
-	}
-
-	return rowsRead, rows.Err()
-}

internal/app/etl/extractors/postgres.go

@@ -1,110 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-type PostgresExtractor struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewPostgresExtractor(db dbwrapper.DbWrapper) etl.Extractor {
-	return &PostgresExtractor{db: db}
-}
-
-func buildExtractQueryPostgres(sourceDbInfo config.SourceTableInfo, columns []models.ColumnType) string {
-	var sbColumns strings.Builder
-
-	if len(columns) == 0 {
-		sbColumns.WriteString("*")
-	} else {
-		for i, col := range columns {
-			if col.Type() == "GEOMETRY" {
-				sbColumns.WriteString(`ST_AsEWKB("`)
-				sbColumns.WriteString(col.Name())
-				sbColumns.WriteString(`") AS "`)
-				sbColumns.WriteString(col.Name())
-				sbColumns.WriteString(`"`)
-			} else {
-				sbColumns.WriteString(`"`)
-				sbColumns.WriteString(col.Name())
-				sbColumns.WriteString(`"`)
-			}
-
-			if i < len(columns)-1 {
-				sbColumns.WriteString(", ")
-			}
-		}
-	}
-
-	return fmt.Sprintf(`SELECT %s FROM "%s"."%s"  ORDER BY "%s" ASC`, sbColumns.String(), sourceDbInfo.Schema, sourceDbInfo.Table, sourceDbInfo.PrimaryKey)
-}
-
-func (postgresEx *PostgresExtractor) Exec(
-	ctx context.Context,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	partition models.Partition,
-	indexPrimaryKey int,
-	chBatchesOut chan<- models.Batch,
-) (int64, error) {
-	query := buildExtractQueryPostgres(tableInfo, columns)
-
-	if partition.HasRange {
-		return 0, errors.New("Batch config not yet supported")
-	}
-
-	var rowsRead int64 = 0
-	rows, err := postgresEx.db.Query(ctx, query)
-	if err != nil {
-		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
-	}
-	defer rows.Close()
-
-	batchRows := make([]models.UnknownRowValues, 0, batchSize)
-
-	for rows.Next() {
-		values, err := rows.Values()
-		if err != nil {
-			return rowsRead, errors.New("Unexpected error reading rows from source")
-		}
-		rowsRead++
-
-		batchRows = append(batchRows, values)
-
-		if len(batchRows) >= batchSize {
-			select {
-			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
-			case <-ctx.Done():
-				return rowsRead, ctx.Err()
-			}
-
-			batchRows = make([]models.UnknownRowValues, 0, batchSize)
-		}
-	}
-
-	if err := rows.Err(); err != nil {
-		return rowsRead, errors.New("Unexpected error reading rows from source")
-	}
-
-	if len(batchRows) > 0 {
-		select {
-		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
-		case <-ctx.Done():
-			return rowsRead, nil
-		}
-	}
-
-	return rowsRead, nil
-}

internal/app/etl/loaders/postgres.go

@@ -1,127 +0,0 @@
-package loaders
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/jackc/pgx/v5/pgconn"
-)
-
-type PostgresLoader struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewPostgresLoader(db dbwrapper.DbWrapper) etl.Loader {
-	return &PostgresLoader{db: db}
-}
-
-func mapSlice[T any, V any](input []T, mapper func(T) V) []V {
-	result := make([]V, len(input))
-
-	for i, v := range input {
-		result[i] = mapper(v)
-	}
-
-	return result
-}
-
-func (postgresLd *PostgresLoader) ProcessBatch(
-	ctx context.Context,
-	tableInfo config.TargetTableInfo,
-	colNames []string,
-	batch models.Batch,
-) (int, error) {
-	_, err := postgresLd.db.SaveMassive(
-		ctx,
-		tableInfo.Schema,
-		tableInfo.Table,
-		colNames,
-		batch.Rows,
-	)
-
-	if err != nil {
-		var pgErr *pgconn.PgError
-		if errors.As(err, &pgErr) {
-			if pgErr.Code == "23505" {
-				return 0, &custom_errors.JobError{
-					ShouldCancelJob: true,
-					Msg:             fmt.Sprintf("Fatal error in table %s.%s", tableInfo.Schema, tableInfo.Table),
-					Prev:            err,
-				}
-			}
-		}
-
-		return 0, &custom_errors.LoaderError{Batch: batch, Msg: err.Error()}
-	}
-
-	return len(batch.Rows), nil
-}
-
-func (postgresLd *PostgresLoader) Exec(
-	ctx context.Context,
-	tableInfo config.TargetTableInfo,
-	columns []models.ColumnType,
-	chBatchesIn <-chan models.Batch,
-	chErrorsOut chan<- custom_errors.LoaderError,
-	chJobErrorsOut chan<- custom_errors.JobError,
-	wgActiveBatches *sync.WaitGroup,
-	rowsLoaded *int64,
-) {
-	colNames := mapSlice(columns, func(col models.ColumnType) string {
-		return col.Name()
-	})
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		case batch, ok := <-chBatchesIn:
-			if !ok {
-				return
-			}
-
-			processedRows, err := postgresLd.ProcessBatch(ctx, tableInfo, colNames, batch)
-
-			if err != nil {
-				var ldError *custom_errors.LoaderError
-				var jobError *custom_errors.JobError
-				if errors.As(err, &ldError) {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- *ldError:
-					}
-				} else if errors.As(err, &jobError) {
-					select {
-					case <-ctx.Done():
-						return
-					case chJobErrorsOut <- *jobError:
-					}
-				} else {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- custom_errors.LoaderError{Batch: batch, Msg: err.Error()}:
-					}
-				}
-
-				continue
-			}
-
-			wgActiveBatches.Done()
-			atomic.AddInt64(rowsLoaded, int64(processedRows))
-		}
-	}
-}

internal/app/etl/loaders/types.go

@@ -1 +0,0 @@
-package loaders

internal/app/etl/table_analyzers/main.go

@@ -1,40 +0,0 @@
-package table_analyzers
-
-import (
-	"context"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-func PartitionRangeGenerator(
-	ctx context.Context,
-	tableAnalyzer etl.TableAnalyzer,
-	tableInfo config.TableInfo,
-	partitionColumn string,
-	rowsPerPartition int64,
-) ([]models.Partition, error) {
-	rowsCount, err := tableAnalyzer.EstimateTotalRows(ctx, tableInfo)
-	if err != nil {
-		return nil, err
-	}
-
-	if rowsCount <= rowsPerPartition {
-		return []models.Partition{{
-			Id:           uuid.New(),
-			HasRange:     false,
-			RetryCounter: 0,
-		}}, nil
-
-	}
-
-	partitionsCount := rowsCount / rowsPerPartition
-	partitions, err := tableAnalyzer.CalculatePartitionRanges(ctx, tableInfo, partitionColumn, partitionsCount)
-	if err != nil {
-		return nil, err
-	}
-
-	return partitions, nil
-}

internal/app/etl/table_analyzers/mssql.go

@@ -1,252 +0,0 @@
-package table_analyzers
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"strings"
-	"time"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-type MssqlTableAnalyzer struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewMssqlTableAnalyzer(db dbwrapper.DbWrapper) etl.TableAnalyzer {
-	return &MssqlTableAnalyzer{db: db}
-}
-
-const mssqlColumnMetadataQuery string = `
-SELECT 
-	c.name AS name,
-	t.name AS user_type,
-	CASE WHEN t.is_user_defined = 0 THEN t.name ELSE bt.name END AS system_type,
-	c.is_nullable AS nullable,
-	c.max_length AS max_length,
-	c.precision AS precision,
-	c.scale AS scale
-FROM sys.columns c
-JOIN sys.types t ON c.user_type_id = t.user_type_id
-LEFT JOIN sys.types bt ON t.is_user_defined = 1 AND bt.user_type_id = t.system_type_id
-JOIN sys.tables st ON c.object_id = st.object_id
-JOIN sys.schemas s ON st.schema_id = s.schema_id
-WHERE s.name = @schema AND st.name = @table AND (c.is_hidden = 0 OR (c.graph_type IS NOT NULL AND c.name LIKE '$%'))
-ORDER BY c.column_id;`
-
-type rawColumnMssql struct {
-	name       string
-	userType   string
-	systemType string
-	nullable   bool
-	maxLength  int64
-	precision  int64
-	scale      int64
-}
-
-func (ta *MssqlTableAnalyzer) systemTypeToUnifiedType(systemType string) string {
-	systemType = strings.ToLower(systemType)
-
-	if systemType == "varchar" || systemType == "char" || systemType == "nvarchar" || systemType == "nchar" || systemType == "text" || systemType == "ntext" {
-		return "STRING"
-	}
-
-	if systemType == "int" || systemType == "int4" || systemType == "integer" || systemType == "smallint" || systemType == "int2" || systemType == "bigint" || systemType == "int8" || systemType == "tinyint" {
-		return "INTEGER"
-	}
-
-	if systemType == "decimal" || systemType == "numeric" {
-		return "DECIMAL"
-	}
-
-	if systemType == "float" || systemType == "real" || systemType == "double precision" {
-		return "FLOAT"
-	}
-
-	if systemType == "bit" || systemType == "boolean" {
-		return "BOOLEAN"
-	}
-
-	if systemType == "date" {
-		return "DATE"
-	}
-	if systemType == "time" || systemType == "time without time zone" {
-		return "TIME"
-	}
-	if systemType == "datetime" || systemType == "datetime2" || systemType == "timestamp" || systemType == "timestamptz" || systemType == "timestamp with time zone" {
-		return "TIMESTAMP"
-	}
-
-	if systemType == "binary" || systemType == "varbinary" || systemType == "image" || systemType == "bytea" {
-		return "BINARY"
-	}
-
-	if systemType == "uniqueidentifier" || systemType == "uuid" {
-		return "UUID"
-	}
-
-	if systemType == "json" {
-		return "JSON"
-	}
-
-	if systemType == "geometry" || systemType == "geography" {
-		return "GEOMETRY"
-	}
-
-	return strings.ToUpper(systemType)
-}
-
-func (ta *MssqlTableAnalyzer) rawColumnToColumnType(rawColumn rawColumnMssql) models.ColumnType {
-	const nullValue int64 = -1
-	stringTypes := map[string]bool{"varchar": true, "char": true, "nvarchar": true, "nchar": true, "text": true, "ntext": true}
-	decimalTypes := map[string]bool{"decimal": true, "numeric": true}
-
-	if stringTypes[rawColumn.systemType] {
-		if rawColumn.systemType == "nvarchar" || rawColumn.systemType == "nchar" {
-			if rawColumn.maxLength > 0 {
-				rawColumn.maxLength = rawColumn.maxLength / 2
-			}
-		}
-
-		rawColumn.precision, rawColumn.scale = nullValue, nullValue
-	} else if decimalTypes[rawColumn.systemType] {
-		rawColumn.maxLength = nullValue
-	} else {
-		rawColumn.maxLength, rawColumn.precision, rawColumn.scale = nullValue, nullValue, nullValue
-	}
-
-	columnType := models.NewColumnType(
-		rawColumn.name,
-		rawColumn.maxLength != nullValue,
-		rawColumn.precision != nullValue || rawColumn.scale != nullValue,
-		rawColumn.userType,
-		rawColumn.systemType,
-		ta.systemTypeToUnifiedType(rawColumn.systemType),
-		rawColumn.nullable,
-		rawColumn.maxLength,
-		rawColumn.precision,
-		rawColumn.scale,
-	)
-
-	return columnType
-}
-
-func (ta *MssqlTableAnalyzer) QueryColumnTypes(
-	ctx context.Context,
-	tableInfo config.TableInfo,
-) ([]models.ColumnType, error) {
-	localCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
-	defer cancel()
-
-	rows, err := ta.db.Query(localCtx, mssqlColumnMetadataQuery, sql.Named("schema", tableInfo.Schema), sql.Named("table", tableInfo.Table))
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var columnTypes []models.ColumnType
-
-	for rows.Next() {
-		var rawColumn rawColumnMssql
-
-		if err := rows.Scan(
-			&rawColumn.name,
-			&rawColumn.userType,
-			&rawColumn.systemType,
-			&rawColumn.nullable,
-			&rawColumn.maxLength,
-			&rawColumn.precision,
-			&rawColumn.scale,
-		); err != nil {
-			return nil, err
-		}
-
-		columnTypes = append(columnTypes, ta.rawColumnToColumnType(rawColumn))
-	}
-
-	return columnTypes, nil
-}
-
-func (ta *MssqlTableAnalyzer) EstimateTotalRows(
-	ctx context.Context,
-	tableInfo config.TableInfo,
-) (int64, error) {
-	query := `
-SELECT SUM(p.rows) AS count
-FROM sys.tables t
-JOIN sys.schemas s ON t.schema_id = s.schema_id
-JOIN sys.partitions p ON t.object_id = p.object_id
-WHERE s.name = @schema AND t.name = @table AND p.index_id IN (0, 1)
-GROUP BY t.name`
-
-	ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Minute)
-	defer cancel()
-
-	var rowsCount int64
-	err := ta.db.QueryRow(ctxTimeout, query, sql.Named("schema", tableInfo.Schema), sql.Named("table", tableInfo.Table)).Scan(&rowsCount)
-	if err != nil {
-		return 0, err
-	}
-
-	return rowsCount, nil
-}
-
-func (ta *MssqlTableAnalyzer) CalculatePartitionRanges(
-	ctx context.Context,
-	tableInfo config.TableInfo,
-	partitionColumn string,
-	maxPartitions int64,
-) ([]models.Partition, error) {
-	query := fmt.Sprintf(`
-SELECT
-	MIN([%s]) AS lower_limit,
-	MAX([%s]) AS upper_limit
-FROM (SELECT [%s], NTILE(@maxPartitions) OVER (ORDER BY [%s]) AS batch_id FROM [%s].[%s]) AS T
-GROUP BY batch_id
-ORDER BY batch_id`,
-		partitionColumn,
-		partitionColumn,
-		partitionColumn,
-		partitionColumn,
-		tableInfo.Schema,
-		tableInfo.Table)
-
-	ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Minute)
-	defer cancel()
-
-	rows, err := ta.db.Query(ctxTimeout, query, sql.Named("maxPartitions", maxPartitions))
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	partitions := make([]models.Partition, 0, maxPartitions)
-
-	for rows.Next() {
-		partition := models.Partition{
-			Id:           uuid.New(),
-			HasRange:     true,
-			RetryCounter: 0,
-			Range: models.PartitionRange{
-				IsMinInclusive: true,
-			},
-		}
-
-		if err := rows.Scan(&partition.Range.Min, &partition.Range.Max); err != nil {
-			return nil, err
-		}
-
-		partitions = append(partitions, partition)
-	}
-
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-
-	return partitions, nil
-}

internal/app/etl/table_analyzers/postgres.go

@@ -1,174 +0,0 @@
-package table_analyzers
-
-import (
-	"context"
-	"strings"
-	"time"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-type PostgresTableAnalyzer struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewPostgresTableAnalyzer(db dbwrapper.DbWrapper) etl.TableAnalyzer {
-	return &PostgresTableAnalyzer{db: db}
-}
-
-const postgresColumnMetadataQuery string = `
-SELECT 
-  c.column_name AS name,
-  c.data_type AS user_type,
-  c.udt_name AS system_type,
-  (CASE WHEN c.is_nullable = 'YES' THEN TRUE ELSE FALSE END) AS nullable,
-  COALESCE(c.character_maximum_length, -1) AS max_length,
-  COALESCE(c.numeric_precision, -1) AS precision,
-  COALESCE(c.numeric_scale, -1) AS scale
-FROM information_schema.columns c
-WHERE c.table_schema = $1 AND c.table_name = $2
-ORDER BY c.ordinal_position;`
-
-type rawColumnPostgres struct {
-	name       string
-	userType   string
-	systemType string
-	nullable   bool
-	maxLength  int64
-	precision  int64
-	scale      int64
-}
-
-func (ta *PostgresTableAnalyzer) systemTypeToUnifiedType(systemType string) string {
-	systemType = strings.ToLower(systemType)
-
-	if systemType == "varchar" || systemType == "char" || systemType == "nvarchar" || systemType == "nchar" || systemType == "text" || systemType == "ntext" {
-		return "STRING"
-	}
-
-	if systemType == "int" || systemType == "int4" || systemType == "integer" || systemType == "smallint" || systemType == "int2" || systemType == "bigint" || systemType == "int8" || systemType == "tinyint" {
-		return "INTEGER"
-	}
-
-	if systemType == "decimal" || systemType == "numeric" {
-		return "DECIMAL"
-	}
-
-	if systemType == "float" || systemType == "real" || systemType == "double precision" {
-		return "FLOAT"
-	}
-
-	if systemType == "bit" || systemType == "boolean" {
-		return "BOOLEAN"
-	}
-
-	if systemType == "date" {
-		return "DATE"
-	}
-	if systemType == "time" || systemType == "time without time zone" {
-		return "TIME"
-	}
-	if systemType == "datetime" || systemType == "datetime2" || systemType == "timestamp" || systemType == "timestamptz" || systemType == "timestamp with time zone" {
-		return "TIMESTAMP"
-	}
-
-	if systemType == "binary" || systemType == "varbinary" || systemType == "image" || systemType == "bytea" {
-		return "BINARY"
-	}
-
-	if systemType == "uniqueidentifier" || systemType == "uuid" {
-		return "UUID"
-	}
-
-	if systemType == "json" {
-		return "JSON"
-	}
-
-	if systemType == "geometry" || systemType == "geography" {
-		return "GEOMETRY"
-	}
-
-	return strings.ToUpper(systemType)
-}
-
-func (ta *PostgresTableAnalyzer) rawColumnToColumnType(rawColumn rawColumnPostgres) models.ColumnType {
-	const nullValue int64 = -1
-	stringTypes := map[string]bool{"varchar": true, "char": true, "text": true}
-	decimalTypes := map[string]bool{"decimal": true, "numeric": true}
-
-	if stringTypes[rawColumn.systemType] {
-		rawColumn.precision, rawColumn.scale = nullValue, nullValue
-	} else if decimalTypes[rawColumn.systemType] {
-		rawColumn.maxLength = nullValue
-	} else {
-		rawColumn.maxLength, rawColumn.precision, rawColumn.scale = nullValue, nullValue, nullValue
-	}
-
-	return models.NewColumnType(
-		rawColumn.name,
-		rawColumn.maxLength != nullValue,
-		rawColumn.precision != nullValue || rawColumn.scale != nullValue,
-		rawColumn.userType,
-		rawColumn.systemType,
-		ta.systemTypeToUnifiedType(rawColumn.systemType),
-		rawColumn.nullable,
-		rawColumn.maxLength,
-		rawColumn.precision,
-		rawColumn.scale,
-	)
-}
-
-func (ta *PostgresTableAnalyzer) QueryColumnTypes(
-	ctx context.Context,
-	tableInfo config.TableInfo,
-) ([]models.ColumnType, error) {
-	localCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
-	defer cancel()
-
-	rows, err := ta.db.Query(localCtx, postgresColumnMetadataQuery, tableInfo.Schema, tableInfo.Table)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var colTypes []models.ColumnType
-
-	for rows.Next() {
-		var column rawColumnPostgres
-
-		if err := rows.Scan(
-			&column.name,
-			&column.userType,
-			&column.systemType,
-			&column.nullable,
-			&column.maxLength,
-			&column.precision,
-			&column.scale,
-		); err != nil {
-			return nil, err
-		}
-
-		colTypes = append(colTypes, ta.rawColumnToColumnType(column))
-	}
-
-	return colTypes, nil
-}
-
-func (ta *PostgresTableAnalyzer) EstimateTotalRows(
-	ctx context.Context,
-	tableInfo config.TableInfo,
-) (int64, error) {
-	return 0, nil
-}
-
-func (ta *PostgresTableAnalyzer) CalculatePartitionRanges(
-	ctx context.Context,
-	tableInfo config.TableInfo,
-	partitionColumn string,
-	maxPartitions int64,
-) ([]models.Partition, error) {
-	return []models.Partition{}, nil
-}

internal/app/etl/transformers/mssql.go

@@ -1,154 +0,0 @@
-package transformers
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"time"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-type MssqlTransformer struct{}
-
-func NewMssqlTransformer() etl.Transformer {
-	return &MssqlTransformer{}
-}
-
-func computeTransformationPlan(columns []models.ColumnType) []etl.ColumnTransformPlan {
-	var plan []etl.ColumnTransformPlan
-
-	for i, col := range columns {
-		switch col.SystemType() {
-		case "uniqueidentifier":
-			plan = append(plan, etl.ColumnTransformPlan{
-				Index: i,
-				Fn: func(v any) (any, error) {
-					if b, ok := v.([]byte); ok && b != nil {
-						return mssqlUuidToBigEndian(b)
-					}
-					return v, nil
-				},
-			})
-
-		case "geometry", "geography":
-			plan = append(plan, etl.ColumnTransformPlan{
-				Index: i,
-				Fn: func(v any) (any, error) {
-					if b, ok := v.([]byte); ok && b != nil {
-						return wkbToEwkbWithSrid(b, 4326)
-					}
-					return v, nil
-				},
-			})
-
-		case "datetime", "datetime2":
-			plan = append(plan, etl.ColumnTransformPlan{
-				Index: i,
-				Fn: func(v any) (any, error) {
-					if t, ok := v.(time.Time); ok {
-						return ensureUTC(t), nil
-					}
-					return v, nil
-				},
-			})
-		}
-	}
-
-	return plan
-}
-
-const processBatchCtxCheck = 4096
-
-func (mssqlTr *MssqlTransformer) ProcessBatch(
-	ctx context.Context,
-	batch *models.Batch,
-	transformationPlan []etl.ColumnTransformPlan,
-) error {
-	for i, rowValues := range batch.Rows {
-		if i%processBatchCtxCheck == 0 {
-			if err := ctx.Err(); err != nil {
-				return err
-			}
-		}
-
-		if rowValues == nil {
-			continue
-		}
-
-		for _, task := range transformationPlan {
-			val := rowValues[task.Index]
-			if val == nil {
-				continue
-			}
-
-			transformed, err := task.Fn(val)
-			if err != nil {
-				return err
-			}
-			rowValues[task.Index] = transformed
-		}
-	}
-
-	return nil
-}
-
-func (mssqlTr *MssqlTransformer) Exec(
-	ctx context.Context,
-	columns []models.ColumnType,
-	chBatchesIn <-chan models.Batch,
-	chBatchesOut chan<- models.Batch,
-	chJobErrorsOut chan<- custom_errors.JobError,
-	wgActiveBatches *sync.WaitGroup,
-) {
-	transformationPlan := computeTransformationPlan(columns)
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-
-		case batch, ok := <-chBatchesIn:
-			if !ok {
-				return
-			}
-
-			if len(transformationPlan) == 0 {
-				select {
-				case chBatchesOut <- batch:
-					wgActiveBatches.Add(1)
-					continue
-				case <-ctx.Done():
-					return
-				}
-			}
-
-			err := mssqlTr.ProcessBatch(ctx, &batch, transformationPlan)
-			if err != nil {
-				if errors.Is(err, ctx.Err()) {
-					return
-				}
-
-				select {
-				case chJobErrorsOut <- custom_errors.JobError{ShouldCancelJob: true, Msg: "Transformation failed", Prev: err}:
-				case <-ctx.Done():
-				}
-				return
-			}
-
-			select {
-			case chBatchesOut <- batch:
-			case <-ctx.Done():
-				return
-			}
-
-			wgActiveBatches.Add(1)
-		}
-	}
-}

internal/app/etl/transformers/types.go

@@ -1 +0,0 @@
-package transformers

internal/app/etl/types.go

@@ -1,85 +0,0 @@
-package etl
-
-import (
-	"context"
-	"sync"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-type Extractor interface {
-	Exec(
-		ctx context.Context,
-		tableInfo config.SourceTableInfo,
-		columns []models.ColumnType,
-		batchSize int,
-		partition models.Partition,
-		indexPrimaryKey int,
-		chBatchesOut chan<- models.Batch,
-	) (int64, error)
-}
-
-type TransformerFunc func(any) (any, error)
-
-type ColumnTransformPlan struct {
-	Index int
-	Fn    TransformerFunc
-}
-
-type Transformer interface {
-	ProcessBatch(
-		ctx context.Context,
-		batch *models.Batch,
-		transformationPlan []ColumnTransformPlan,
-	) error
-
-	Exec(
-		ctx context.Context,
-		columns []models.ColumnType,
-		chBatchesIn <-chan models.Batch,
-		chBactchesOut chan<- models.Batch,
-		chJobErrorsOut chan<- custom_errors.JobError,
-		wgActiveBatches *sync.WaitGroup,
-	)
-}
-
-type Loader interface {
-	ProcessBatch(
-		ctx context.Context,
-		tableInfo config.TargetTableInfo,
-		colNames []string,
-		batch models.Batch,
-	) (int, error)
-
-	Exec(
-		ctx context.Context,
-		tableInfo config.TargetTableInfo,
-		columns []models.ColumnType,
-		chBatchesIn <-chan models.Batch,
-		chErrorsOut chan<- custom_errors.LoaderError,
-		chJobErrorsOut chan<- custom_errors.JobError,
-		wgActiveBatches *sync.WaitGroup,
-		rowsLoaded *int64,
-	)
-}
-
-type TableAnalyzer interface {
-	QueryColumnTypes(
-		ctx context.Context,
-		tableInfo config.TableInfo,
-	) ([]models.ColumnType, error)
-
-	EstimateTotalRows(
-		ctx context.Context,
-		tableInfo config.TableInfo,
-	) (int64, error)
-
-	CalculatePartitionRanges(
-		ctx context.Context,
-		tableInfo config.TableInfo,
-		partitionColumn string,
-		maxPartitions int64,
-	) ([]models.Partition, error)
-}

internal/app/models/main.go

@@ -1,27 +0,0 @@
-package models
-
-import "github.com/google/uuid"
-
-type UnknownRowValues = []any
-
-type Batch struct {
-	Id           uuid.UUID
-	PartitionId  uuid.UUID
-	Rows         []UnknownRowValues
-	RetryCounter int
-}
-
-type PartitionRange struct {
-	Min            int64
-	Max            int64
-	IsMinInclusive bool
-	IsMaxInclusive bool
-}
-
-type Partition struct {
-	Id           uuid.UUID
-	ParentId     uuid.UUID
-	Range        PartitionRange
-	HasRange     bool
-	RetryCounter int
-}

scripts/config-parser/main.go

@@ -1,17 +1,131 @@
 package main
 
 import (
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	log "github.com/sirupsen/logrus"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/goccy/go-yaml"
 )
 
-func main() {
-	log.SetLevel(log.DebugLevel)
+// Estructuras para mapear el YAML
+type RetryConfig struct {
+	Attempts int `yaml:"attempts"`
+}
 
-	migrationConfig, err := config.ReadMigrationConfig()
+type DBInfo struct {
+	Schema     string `yaml:"schema"`
+	Table      string `yaml:"table"`
+	PrimaryKey string `yaml:"primary_key,omitempty"` // omitempty si no siempre existe
+}
+
+// JobSettings contiene los campos que se comparten entre 'defaults' y cada 'job'
+type JobSettings struct {
+	MaxExtractors  *int         `yaml:"max_extractors"`
+	MaxLoaders     *int         `yaml:"max_loaders"`
+	QueueSize      *int         `yaml:"queue_size"`
+	ChunkSize      *int         `yaml:"chunk_size"`
+	ChunksPerBatch *int         `yaml:"chunks_per_batch"`
+	TruncateTarget *bool        `yaml:"truncate_target"`
+	TruncateMethod *string      `yaml:"truncate_method"`
+	Retry          *RetryConfig `yaml:"retry"`
+}
+
+type Job struct {
+	Name    string   `yaml:"name"`
+	Enabled bool     `yaml:"enabled"`
+	Source  DBInfo   `yaml:"source"`
+	Target  DBInfo   `yaml:"target"`
+	PreSQL  []string `yaml:"pre_sql"`
+	PostSQL []string `yaml:"post_sql"`
+	// Incrustamos los settings para permitir los overrides
+	JobSettings `yaml:",inline"`
+}
+
+type Config struct {
+	MaxParallelWorkers int         `yaml:"max_parallel_workers"`
+	Defaults           JobSettings `yaml:"defaults"`
+	Jobs               []Job       `yaml:"jobs"`
+}
+
+func main() {
+	yamlFile, err := os.ReadFile("config.yaml")
 	if err != nil {
-		log.Fatalf("error leyendo configuracion: %v", err)
+		log.Fatalf("Error leyendo archivo: %v", err)
 	}
 
-	log.Debugf("Config: %+v", migrationConfig)
+	var config Config
+	err = yaml.Unmarshal(yamlFile, &config)
+	if err != nil {
+		log.Fatalf("Error parseando YAML: %v", err)
+	}
+
+	fmt.Printf("Configuración cargada. Trabajos: %d\n", len(config.Jobs))
+
+	for i, job := range config.Jobs {
+		jobPtr := &config.Jobs[i]
+
+		if job.MaxExtractors == nil {
+			jobPtr.MaxExtractors = config.Defaults.MaxExtractors
+		}
+		if job.MaxLoaders == nil {
+			jobPtr.MaxLoaders = config.Defaults.MaxLoaders
+		}
+		if job.QueueSize == nil {
+			jobPtr.QueueSize = config.Defaults.QueueSize
+		}
+		if job.ChunkSize == nil {
+			jobPtr.ChunkSize = config.Defaults.ChunkSize
+		}
+		if job.ChunksPerBatch == nil {
+			jobPtr.ChunksPerBatch = config.Defaults.ChunksPerBatch
+		}
+		if job.TruncateTarget == nil {
+			jobPtr.TruncateTarget = config.Defaults.TruncateTarget
+		}
+		if job.TruncateMethod == nil {
+			jobPtr.TruncateMethod = config.Defaults.TruncateMethod
+		}
+		if job.Retry == nil {
+			jobPtr.Retry = config.Defaults.Retry
+		}
+	}
+
+	printConfig(config)
+}
+
+func printConfig(config Config) {
+	fmt.Println("Max parallel workers: ", config.MaxParallelWorkers)
+
+	fmt.Println("Defaults:")
+	fmt.Printf("\tMaxExtractors: %v\n", *config.Defaults.MaxExtractors)
+	fmt.Printf("\tMaxLoaders: %v\n", *config.Defaults.MaxLoaders)
+	fmt.Printf("\tQueueSize: %v\n", *config.Defaults.QueueSize)
+	fmt.Printf("\tChunkSize: %v\n", *config.Defaults.ChunkSize)
+	fmt.Printf("\tChunksPerBatch: %v\n", *config.Defaults.ChunksPerBatch)
+	fmt.Printf("\tTruncateTarget: %v\n", *config.Defaults.TruncateTarget)
+	fmt.Printf("\tTruncateMethod: %v\n", *config.Defaults.TruncateMethod)
+	fmt.Printf("\tRetry: %v\n", *config.Defaults.Retry)
+
+	fmt.Println("Jobs:")
+	for i, job := range config.Jobs {
+		fmt.Printf("Job Name: %v\n", job.Name)
+		fmt.Printf("\tEnabled: %v\n", job.Enabled)
+		fmt.Printf("\tSource: %v\n", job.Source)
+		fmt.Printf("\tTarget: %v\n", job.Target)
+		fmt.Printf("\tMaxExtractors: %v\n", *job.MaxExtractors)
+		fmt.Printf("\tMaxLoaders: %v\n", *job.MaxLoaders)
+		fmt.Printf("\tQueueSize: %v\n", *job.QueueSize)
+		fmt.Printf("\tChunkSize: %v\n", *job.ChunkSize)
+		fmt.Printf("\tChunksPerBatch: %v\n", *job.ChunksPerBatch)
+		fmt.Printf("\tTruncateTarget: %v\n", *job.TruncateTarget)
+		fmt.Printf("\tTruncateMethod: %v\n", *job.TruncateMethod)
+		fmt.Printf("\tRetry: %v\n", *job.Retry)
+		fmt.Printf("\tPreSQL: %v\n", job.PreSQL)
+		fmt.Printf("\tPostSQL: %v\n", job.PostSQL)
+
+		if i >= 2 {
+			fmt.Println("Skipping remaining jobs...")
+		}
+	}
 }

scripts/pg-info-test/main.go

@@ -8,32 +8,13 @@ import (
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/jackc/pgx/v5/pgxpool"
 	log "github.com/sirupsen/logrus"
 )
 
-func Connect(ctx context.Context, dbURL string) (*pgxpool.Pool, error) {
-	pool, err := pgxpool.New(ctx, dbURL)
-	if err != nil {
-		return nil, fmt.Errorf("unable to connect to database: %w", err)
-	}
-
-	if err := pool.Ping(ctx); err != nil {
-		pool.Close()
-		return nil, fmt.Errorf("unable to ping database: %w", err)
-	}
-
-	return pool, nil
-}
-
-func Close(pool *pgxpool.Pool) {
-	if pool != nil {
-		pool.Close()
-	}
-}
-
 func main() {
 	log.SetFormatter(&log.TextFormatter{
 		FullTimestamp:   true,
@@ -46,8 +27,8 @@ func main() {
 	ctxSource, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 
-	sourcePool, err := Connect(ctxSource, config.App.SourceDbUrl)
-	defer Close(sourcePool)
+	sourcePool, err := db.Connect(ctxSource, config.App.SourceDbUrl)
+	defer db.Close(sourcePool)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -56,8 +37,8 @@ func main() {
 	ctxTarget, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 
-	targetPool, err := Connect(ctxTarget, config.App.TargetDbUrl)
-	defer Close(targetPool)
+	targetPool, err := db.Connect(ctxTarget, config.App.TargetDbUrl)
+	defer db.Close(targetPool)
 	if err != nil {
 		log.Fatal(err)
 	}