feat: update target database type to sqlserver and clean up unused code in mssql.go

.gitignore

@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+bin/
 
 # Test binary, built with `go test -c`
 *.test

Makefile

@@ -0,0 +1,80 @@
+.PHONY: build build-linux build-windows build-all clean help
+
+# Variables
+BINARY_NAME=go-migrate
+CMD_PATH=./cmd/go_migrate
+OUTPUT_DIR=bin
+VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
+BUILD_TIME=$(shell date -u '+%Y-%m-%d_%H:%M:%S')
+GIT_COMMIT=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+
+# Flags de compilación
+LD_FLAGS=-ldflags="-s -w -X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT)"
+
+# Default: compilar para el SO actual
+build: build-$(OS)
+
+ifeq ($(OS),Windows_NT)
+build-native: build-windows
+else
+build-native: build-linux
+endif
+
+# Compilar para Linux (sin CGO para máxima compatibilidad)
+build-linux:
+	@echo "Compilando para Linux..."
+	@mkdir -p $(OUTPUT_DIR)
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+		$(LD_FLAGS) \
+		-o $(OUTPUT_DIR)/$(BINARY_NAME)-linux-amd64 \
+		$(CMD_PATH)
+	@echo "Binario creado: $(OUTPUT_DIR)/$(BINARY_NAME)-linux-amd64"
+
+# Compilar para Windows
+build-windows:
+	@echo "Compilando para Windows..."
+	@mkdir -p $(OUTPUT_DIR)
+	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build \
+		$(LD_FLAGS) \
+		-o $(OUTPUT_DIR)/$(BINARY_NAME)-windows-amd64.exe \
+		$(CMD_PATH)
+	@echo "Binario creado: $(OUTPUT_DIR)/$(BINARY_NAME)-windows-amd64.exe"
+
+# Compilar para ambas plataformas
+build-all: build-linux build-windows
+	@echo ""
+	@echo "Binarios compilados:"
+	@ls -lh $(OUTPUT_DIR)/$(BINARY_NAME)*
+
+# Compilar para Linux arm64 (opcional, para Raspberry Pi, etc.)
+build-linux-arm64:
+	@echo "Compilando para Linux ARM64..."
+	@mkdir -p $(OUTPUT_DIR)
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build \
+		$(LD_FLAGS) \
+		-o $(OUTPUT_DIR)/$(BINARY_NAME)-linux-arm64 \
+		$(CMD_PATH)
+	@echo "Binario creado: $(OUTPUT_DIR)/$(BINARY_NAME)-linux-arm64"
+
+# Limpiar binarios
+clean:
+	@echo "Limpiando binarios..."
+	@rm -rf $(OUTPUT_DIR)
+	@echo "Limpieza completada"
+
+# Ayuda
+help:
+	@echo "Comandos disponibles:"
+	@echo ""
+	@echo "  make build              - Compilar para el SO actual (Linux/Windows)"
+	@echo "  make build-linux        - Compilar para Linux x86_64"
+	@echo "  make build-windows      - Compilar para Windows x86_64"
+	@echo "  make build-linux-arm64  - Compilar para Linux ARM64 (opcional)"
+	@echo "  make build-all          - Compilar para Linux y Windows"
+	@echo "  make clean              - Eliminar binarios compilados"
+	@echo "  make help               - Mostrar esta ayuda"
+	@echo ""
+	@echo "Ejemplos de uso:"
+	@echo "  make build-all          # Crear binarios para ambas plataformas"
+	@echo "  make build-linux OS=    # Crear solo para Linux"
+	@echo ""

cmd/go_migrate/connect.go

@@ -1,77 +0,0 @@
-package main
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"sync"
-	"time"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"github.com/jackc/pgx/v5/pgxpool"
-	_ "github.com/microsoft/go-mssqldb"
-	log "github.com/sirupsen/logrus"
-)
-
-func connectToSqlServer() (*sql.DB, error) {
-	db, err := sql.Open("sqlserver", config.App.SourceDbUrl)
-	if err != nil {
-		return nil, fmt.Errorf("Unable to connect to sqlserver: %w", err)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	defer cancel()
-
-	if err := db.PingContext(ctx); err != nil {
-		return nil, fmt.Errorf("Unable to ping sqlserver: %w", err)
-	}
-
-	return db, nil
-}
-
-func connectToPostgres() (*pgxpool.Pool, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	defer cancel()
-
-	pool, err := pgxpool.New(ctx, config.App.TargetDbUrl)
-	if err != nil {
-		return nil, fmt.Errorf("Unable to connect to postgres: %w", err)
-	}
-
-	if err := pool.Ping(ctx); err != nil {
-		pool.Close()
-		return nil, fmt.Errorf("Unable to ping postgres: %w", err)
-	}
-
-	return pool, nil
-}
-
-func connectToDatabases() (*sql.DB, *pgxpool.Pool, error) {
-	var sourceDbErr, targetDbErr error
-	var sourceDb *sql.DB
-	var targetDb *pgxpool.Pool
-	var wg sync.WaitGroup
-
-	wg.Go(func() {
-		sourceDb, sourceDbErr = connectToSqlServer()
-		if sourceDbErr != nil {
-			log.Error("Unable to connect to source db: ", sourceDbErr)
-		}
-	})
-
-	wg.Go(func() {
-		targetDb, targetDbErr = connectToPostgres()
-		if targetDbErr != nil {
-			log.Error("Unable to connect to target db: ", targetDbErr)
-		}
-	})
-
-	wg.Wait()
-
-	if sourceDbErr != nil || targetDbErr != nil {
-		return nil, nil, errors.New("Unable to connect to databases")
-	}
-
-	return sourceDb, targetDb, nil
-}

cmd/go_migrate/main.go

@@ -2,18 +2,18 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"sync"
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractors"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loaders"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformers"
-	"github.com/jackc/pgx/v5/pgxpool"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 )
 
 func main() {
@@ -33,11 +33,33 @@ func main() {
 
 	log.Info("=== Starting migration ===")
 
-	sourceDb, targetDb, connError := connectToDatabases()
+	var wgConnect errgroup.Group
-	if connError != nil {
+	var sourceDb, targetDb dbwrapper.DbWrapper
-		log.Fatal("Connection error: ", connError)
+
+	wgConnect.Go(func() error {
+		var err error
+		sourceDb, err = connectWithTimeout(ctx, migrationConfig.SourceDbType, config.App.SourceDbUrl, 20*time.Second)
+		if err != nil {
+			return err
 		}
 
+		return nil
+	})
+
+	wgConnect.Go(func() error {
+		var err error
+		targetDb, err = connectWithTimeout(ctx, migrationConfig.TargetDbType, config.App.TargetDbUrl, 20*time.Second)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err := wgConnect.Wait(); err != nil {
+		log.Error("Connection error: ", err)
+		return
+	}
 	defer sourceDb.Close()
 	defer targetDb.Close()
 
@@ -70,14 +92,14 @@ func main() {
 
 func processMigrationJobs(
 	ctx context.Context,
-	sourceDb *sql.DB,
+	sourceDb dbwrapper.DbWrapper,
-	targetDb *pgxpool.Pool,
+	targetDb dbwrapper.DbWrapper,
 	jobs []config.Job,
 	maxParallelWorkers int,
-) []JobResult {
+) []models.JobResult {
 	if len(jobs) == 0 {
 		log.Info("No migration jobs configured")
-		return []JobResult{}
+		return []models.JobResult{}
 	}
 
 	if maxParallelWorkers <= 0 {
@@ -90,16 +112,15 @@ func processMigrationJobs(
 
 	log.Infof("Starting migration with %d parallel worker(s)", maxParallelWorkers)
 
-	chJobResults := make(chan JobResult, len(jobs))
+	chJobResults := make(chan models.JobResult, len(jobs))
 	chJobs := make(chan config.Job, len(jobs))
 	var wgJobs sync.WaitGroup
 
-	targetDbWrapper := db.NewPostgresDbWrapper(targetDb)
 	sourceTableAnalyzer := table_analyzers.NewMssqlTableAnalyzer(sourceDb)
 	targetTableAnalyzer := table_analyzers.NewPostgresTableAnalyzer(targetDb)
 	extractor := extractors.NewMssqlExtractor(sourceDb)
 	transformer := transformers.NewMssqlTransformer()
-	loader := loaders.NewPostgresLoader(targetDb)
+	loader := loaders.NewGenericLoader(targetDb)
 
 	for i := range maxParallelWorkers {
 		wgJobs.Go(func() {
@@ -107,7 +128,7 @@ func processMigrationJobs(
 				log.Infof("[worker %d] >>> Processing job: %s.%s <<<", i, job.SourceTable.Schema, job.SourceTable.Table)
 				res := processMigrationJob(
 					ctx,
-					targetDbWrapper,
+					targetDb,
 					sourceTableAnalyzer,
 					targetTableAnalyzer,
 					extractor,
@@ -131,10 +152,26 @@ func processMigrationJobs(
 		close(chJobResults)
 	}()
 
-	var finalResults []JobResult
+	var finalResults []models.JobResult
 	for res := range chJobResults {
 		finalResults = append(finalResults, res)
 	}
 
 	return finalResults
 }
+
+func connectWithTimeout(ctx context.Context, dbType string, dbUrl string, timeout time.Duration) (dbwrapper.DbWrapper, error) {
+	localCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	sourceDb, err := dbwrapper.New(dbType)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = sourceDb.Connect(localCtx, dbUrl); err != nil {
+		return nil, err
+	}
+
+	return sourceDb, nil
+}

cmd/go_migrate/metrics.go

@@ -1,13 +0,0 @@
-package main
-
-import "time"
-
-type JobResult struct {
-	JobName    string
-	StartTime  time.Time
-	Duration   time.Duration
-	RowsRead   int64
-	RowsLoaded int64
-	RowsFailed int64
-	Error      error
-}

cmd/go_migrate/process.go

@@ -2,13 +2,14 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
@@ -18,19 +19,18 @@ import (
 
 func processMigrationJob(
 	ctx context.Context,
-	// sourceDbWrapper db.DbWrapper,
+	targetDbWrapper dbwrapper.DbWrapper,
-	targetDbWrapper db.DbWrapper,
 	sourceTableAnalyzer etl.TableAnalyzer,
 	targetTableAnalyzer etl.TableAnalyzer,
 	extractor etl.Extractor,
 	transformer etl.Transformer,
 	loader etl.Loader,
 	job config.Job,
-) JobResult {
+) models.JobResult {
 	localCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	result := JobResult{
+	result := models.JobResult{
 		JobName:   job.Name,
 		StartTime: time.Now(),
 	}
@@ -234,5 +234,9 @@ func processMigrationJob(
 	result.RowsLoaded = atomic.LoadInt64(&rowsLoaded)
 	result.RowsFailed = atomic.LoadInt64(&rowsFailed)
 
+	if result.RowsRead != result.RowsLoaded {
+		result.Error = fmt.Errorf("Row count mismatch: extracted %d rows but loaded %d rows (failed: %d)", result.RowsRead, result.RowsLoaded, result.RowsFailed)
+	}
+
 	return result
 }

config.yaml

@@ -1,6 +1,6 @@
 max_parallel_workers: 4
 source_db_type: sqlserver
-target_db_type: postgres
+target_db_type: sqlserver
 
 defaults:
   max_extractors: 2
@@ -30,6 +30,12 @@ jobs:
       table: MANZANA
     pre_sql: 
       - 'SELECT 1'
+      # - 'TRUNCATE TABLE "Cartografia"."MANZANA"'
+    range:
+      min: 1000000
+      max: 2000000
+      is_min_inclusive: false
+      is_max_inclusive: true
 
   - name: red_puerto
     enabled: true
@@ -42,5 +48,6 @@ jobs:
       table: PUERTO
     pre_sql: 
       - 'SELECT 1'
+      # - 'TRUNCATE TABLE "Red"."PUERTO"'
     post_sql: 
       - "SELECT 1"

internal/app/config/migration.go

@@ -50,6 +50,12 @@ type Job struct {
 	PreSQL      []string        `yaml:"pre_sql"`
 	PostSQL     []string        `yaml:"post_sql"`
 	JobConfig   `yaml:",inline"`
+	Range       struct {
+		Min            int64 `yaml:"min"`
+		Max            int64 `yaml:"max"`
+		IsMinInclusive bool  `yaml:"is_min_inclusive"`
+		IsMaxInclusive bool  `yaml:"is_max_inclusive"`
+	}
 }
 
 type MigrationConfig struct {
@@ -76,6 +82,8 @@ func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
 
 	c.MaxParallelWorkers = raw.MaxParallelWorkers
 	c.Defaults = raw.Defaults
+	c.SourceDbType = raw.SourceDbType
+	c.TargetDbType = raw.TargetDbType
 	c.Defaults.RowsPerPartition = int64(raw.Defaults.BatchSize * raw.Defaults.BatchesPerPartition)
 
 	for _, node := range raw.Jobs {

internal/app/custom_errors/extractor.error.go

@@ -103,8 +103,8 @@ func ExtractorErrorHandler(
 			if err.HasLastId {
 				newPartition.ParentId = err.Partition.Id
 				newPartition.Id = uuid.New()
-				newPartition.LowerLimit = err.LastId
+				newPartition.Range.Min = err.LastId
-				newPartition.IsLowerLimitInclusive = false
+				newPartition.Range.IsMinInclusive = false
 			}
 
 			requeueWithBackoff(ctx, delay, func() {

internal/app/db-wrapper/main.go

@@ -0,0 +1,19 @@
+package dbwrapper
+
+import "fmt"
+
+type Factory func() DbWrapper
+
+var drivers = make(map[string]Factory)
+
+func Register(name string, factory Factory) {
+	drivers[name] = factory
+}
+
+func New(driverType string) (DbWrapper, error) {
+	factory, ok := drivers[driverType]
+	if !ok {
+		return nil, fmt.Errorf("driver not yet supported: %s", driverType)
+	}
+	return factory(), nil
+}

internal/app/db-wrapper/mssql.go

@@ -0,0 +1,176 @@
+package dbwrapper
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	mssql "github.com/microsoft/go-mssqldb"
+)
+
+func init() {
+	Register("sqlserver", func() DbWrapper {
+		return &mssqlDbWrapper{dialect: "sqlserver"}
+	})
+}
+
+type mssqlRowResult struct {
+	row *sql.Row
+}
+
+func (mr *mssqlRowResult) Scan(dest ...any) error {
+	return mr.row.Scan(dest...)
+}
+
+type mssqlRowsResult struct {
+	columns []string
+	rows    *sql.Rows
+}
+
+func (mr *mssqlRowsResult) Close() error {
+	return mr.rows.Close()
+}
+
+func (mr *mssqlRowsResult) Columns() ([]string, error) {
+	if mr.columns != nil {
+		return mr.columns, nil
+	}
+
+	return mr.rows.Columns()
+}
+
+func (mr *mssqlRowsResult) Err() error {
+	return mr.rows.Err()
+}
+
+func (mr *mssqlRowsResult) Next() bool {
+	return mr.rows.Next()
+}
+
+func (mr *mssqlRowsResult) Scan(dest ...any) error {
+	return mr.rows.Scan(dest...)
+}
+
+func (mr *mssqlRowsResult) Values() ([]any, error) {
+	columns, err := mr.Columns()
+	if err != nil {
+		return nil, err
+	}
+
+	rowValues := make([]any, len(columns))
+	scanArgs := make([]any, len(columns))
+	for i := range rowValues {
+		scanArgs[i] = &rowValues[i]
+	}
+
+	if err := mr.rows.Scan(scanArgs...); err != nil {
+		return nil, err
+	}
+
+	return rowValues, nil
+}
+
+type mssqlDbWrapper struct {
+	db      *sql.DB
+	dialect string
+}
+
+func (mw *mssqlDbWrapper) Connect(ctx context.Context, dbUrl string) error {
+	db, err := sql.Open("sqlserver", dbUrl)
+	if err != nil {
+		return err
+	}
+
+	if err := db.PingContext(ctx); err != nil {
+		if err := db.Close(); err != nil {
+			return err
+		}
+		return err
+	}
+
+	mw.db = db
+	return nil
+}
+
+func (mw *mssqlDbWrapper) Close() error {
+	return mw.db.Close()
+}
+
+func (mw *mssqlDbWrapper) Exec(ctx context.Context, query string, args ...any) (ExecResult, error) {
+	result, execErr := mw.db.ExecContext(ctx, query, args...)
+	if execErr != nil {
+		return ExecResult{}, execErr
+	}
+
+	affectedRows, err := result.RowsAffected()
+	if err != nil {
+		return ExecResult{}, err
+	}
+
+	return ExecResult{AffectedRows: affectedRows}, nil
+}
+
+func (mw *mssqlDbWrapper) GetDialect() string {
+	return mw.dialect
+}
+
+func (mw *mssqlDbWrapper) Query(ctx context.Context, query string, args ...any) (RowsResult, error) {
+	rows, err := mw.db.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &mssqlRowsResult{columns: nil, rows: rows}, nil
+}
+
+func (mw *mssqlDbWrapper) QueryRow(ctx context.Context, query string, args ...any) RowResult {
+	row := mw.db.QueryRowContext(ctx, query, args...)
+	return &mssqlRowResult{row: row}
+}
+
+func (mw *mssqlDbWrapper) SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error) {
+	tx, err := mw.db.BeginTx(ctx, nil)
+	if err != nil {
+		return 0, err
+	}
+
+	fullTableName := fmt.Sprintf("[%s].[%s]", schema, table)
+
+	stmt, err := tx.PrepareContext(ctx, mssql.CopyIn(fullTableName, mssql.BulkOptions{}, columnNames...))
+	if err != nil {
+		tx.Rollback()
+		return 0, err
+	}
+
+	for _, row := range rows {
+		_, err = stmt.ExecContext(ctx, row...)
+		if err != nil {
+			stmt.Close()
+			tx.Rollback()
+			return 0, err
+		}
+	}
+
+	result, err := stmt.ExecContext(ctx)
+	if err != nil {
+		stmt.Close()
+		tx.Rollback()
+		return 0, err
+	}
+
+	if err := stmt.Close(); err != nil {
+		tx.Rollback()
+		return 0, err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return 0, err
+	}
+
+	rowsAffected, raErr := result.RowsAffected()
+	if raErr != nil {
+		return 0, nil
+	}
+
+	return rowsAffected, nil
+}

internal/app/db-wrapper/postgres.go

@@ -0,0 +1,128 @@
+package dbwrapper
+
+import (
+	"context"
+	"errors"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+func init() {
+	Register("postgres", func() DbWrapper {
+		return &postgresDbWrapper{dialect: "postgres"}
+	})
+}
+
+type postgresRowResult struct {
+	row pgx.Row
+}
+
+func (pr *postgresRowResult) Scan(dest ...any) error {
+	return pr.row.Scan(dest...)
+}
+
+type postgresRowsResult struct {
+	columns []string
+	rows    pgx.Rows
+}
+
+func (pr *postgresRowsResult) Close() error {
+	pr.rows.Close()
+	return nil
+}
+
+func (pr *postgresRowsResult) Columns() ([]string, error) {
+	if pr.columns != nil {
+		return pr.columns, nil
+	}
+
+	rawColumns := pr.rows.FieldDescriptions()
+	if rawColumns == nil {
+		return nil, errors.New("error retrieving columns")
+	}
+
+	columns := make([]string, 0, len(rawColumns))
+	for _, rc := range rawColumns {
+		columns = append(columns, rc.Name)
+	}
+
+	return columns, nil
+}
+
+func (pr *postgresRowsResult) Err() error {
+	return pr.rows.Err()
+}
+
+func (pr *postgresRowsResult) Next() bool {
+	return pr.rows.Next()
+}
+
+func (pr *postgresRowsResult) Scan(dest ...any) error {
+	return pr.rows.Scan(dest...)
+}
+
+func (pr *postgresRowsResult) Values() ([]any, error) {
+	return pr.rows.Values()
+}
+
+type postgresDbWrapper struct {
+	db      *pgxpool.Pool
+	dialect string
+}
+
+func (pw *postgresDbWrapper) Connect(ctx context.Context, dbUrl string) error {
+	pool, err := pgxpool.New(ctx, dbUrl)
+	if err != nil {
+		return err
+	}
+
+	if err := pool.Ping(ctx); err != nil {
+		pool.Close()
+		return err
+	}
+
+	pw.db = pool
+	return nil
+}
+
+func (pw *postgresDbWrapper) Close() error {
+	pw.db.Close()
+	return nil
+}
+
+func (pw *postgresDbWrapper) Exec(ctx context.Context, query string, args ...any) (ExecResult, error) {
+	result, err := pw.db.Exec(ctx, query, args...)
+	if err != nil {
+		return ExecResult{}, err
+	}
+
+	return ExecResult{AffectedRows: result.RowsAffected()}, nil
+}
+
+func (pw *postgresDbWrapper) GetDialect() string {
+	return pw.dialect
+}
+
+func (pw *postgresDbWrapper) Query(ctx context.Context, query string, args ...any) (RowsResult, error) {
+	rows, err := pw.db.Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &postgresRowsResult{columns: nil, rows: rows}, nil
+}
+
+func (pw *postgresDbWrapper) QueryRow(ctx context.Context, query string, args ...any) RowResult {
+	row := pw.db.QueryRow(ctx, query, args...)
+	return &postgresRowResult{row: row}
+}
+
+func (pw *postgresDbWrapper) SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error) {
+	affectedRows, err := pw.db.CopyFrom(ctx, pgx.Identifier{schema, table}, columnNames, pgx.CopyFromRows(rows))
+	if err != nil {
+		return 0, err
+	}
+
+	return affectedRows, nil
+}

internal/app/db-wrapper/types.go

@@ -0,0 +1,35 @@
+package dbwrapper
+
+import (
+	"context"
+	"errors"
+)
+
+var MethodNotSupported error = errors.New("Method not supported by driver... yet :P")
+
+type ExecResult struct {
+	AffectedRows int64
+}
+
+type RowsResult interface {
+	Close() error
+	Columns() ([]string, error)
+	Err() error
+	Next() bool
+	Scan(dest ...any) error
+	Values() ([]any, error)
+}
+
+type RowResult interface {
+	Scan(dest ...any) error
+}
+
+type DbWrapper interface {
+	Close() error
+	Connect(ctx context.Context, dbUrl string) error
+	Exec(ctx context.Context, query string, args ...any) (ExecResult, error)
+	GetDialect() string
+	Query(ctx context.Context, query string, args ...any) (RowsResult, error)
+	QueryRow(ctx context.Context, query string, args ...any) RowResult
+	SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error)
+}

internal/app/db/mssql.go

@@ -1,30 +0,0 @@
-package db
-
-import (
-	"context"
-	"database/sql"
-)
-
-type MssqlDbWrapper struct {
-	db *sql.DB
-}
-
-func NewMssqlDbWrapper(db *sql.DB) DbWrapper {
-	return &MssqlDbWrapper{db: db}
-}
-
-func (wrapper *MssqlDbWrapper) Exec(ctx context.Context, query string, args ...any) (DbWrapperResult, error) {
-	result, execErr := wrapper.db.ExecContext(ctx, query, args...)
-	if execErr != nil {
-		return DbWrapperResult{}, execErr
-	}
-
-	affectedRows, err := result.RowsAffected()
-	if err != nil {
-		return DbWrapperResult{}, err
-	}
-
-	return DbWrapperResult{
-		AffectedRows: affectedRows,
-	}, nil
-}

internal/app/db/postgres.go

@@ -1,47 +0,0 @@
-package db
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/jackc/pgx/v5/pgxpool"
-)
-
-func Connect(ctx context.Context, dbURL string) (*pgxpool.Pool, error) {
-	pool, err := pgxpool.New(ctx, dbURL)
-	if err != nil {
-		return nil, fmt.Errorf("unable to connect to database: %w", err)
-	}
-
-	if err := pool.Ping(ctx); err != nil {
-		pool.Close()
-		return nil, fmt.Errorf("unable to ping database: %w", err)
-	}
-
-	return pool, nil
-}
-
-func Close(pool *pgxpool.Pool) {
-	if pool != nil {
-		pool.Close()
-	}
-}
-
-type PostgresDbWrapper struct {
-	db *pgxpool.Pool
-}
-
-func NewPostgresDbWrapper(db *pgxpool.Pool) DbWrapper {
-	return &PostgresDbWrapper{db: db}
-}
-
-func (wrapper *PostgresDbWrapper) Exec(ctx context.Context, query string, args ...any) (DbWrapperResult, error) {
-	result, err := wrapper.db.Exec(ctx, query, args...)
-	if err != nil {
-		return DbWrapperResult{}, err
-	}
-
-	return DbWrapperResult{
-		AffectedRows: result.RowsAffected(),
-	}, nil
-}

internal/app/db/types.go

@@ -1,11 +0,0 @@
-package db
-
-import "context"
-
-type DbWrapperResult struct {
-	AffectedRows int64
-}
-
-type DbWrapper interface {
-	Exec(ctx context.Context, query string, args ...any) (DbWrapperResult, error)
-}

internal/app/etl/extractors/mssql.go

@@ -13,16 +13,17 @@ import (
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/convert"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	"github.com/google/uuid"
 )
 
 type MssqlExtractor struct {
-	db *sql.DB
+	db dbwrapper.DbWrapper
 }
 
-func NewMssqlExtractor(db *sql.DB) etl.Extractor {
+func NewMssqlExtractor(db dbwrapper.DbWrapper) etl.Extractor {
 	return &MssqlExtractor{db: db}
 }
 
@@ -42,9 +43,9 @@ func buildExtractQueryMssql(
 		for i, col := range columns {
 			fmt.Fprintf(&sbQuery, "[%s]", col.Name())
 
-			if col.Type() == "GEOMETRY" {
+			// if col.Type() == "GEOMETRY" {
-				fmt.Fprintf(&sbQuery, ".STAsBinary() AS [%s]", col.Name())
+			// 	fmt.Fprintf(&sbQuery, ".STAsBinary() AS [%s]", col.Name())
-			}
+			// }
 
 			if i < len(columns)-1 {
 				sbQuery.WriteString(", ")
@@ -73,14 +74,14 @@ func buildExtractQueryMssql(
 func errorFromLastRow(
 	lastRow models.UnknownRowValues,
 	indexPrimaryKey int,
-	partition *models.Partition,
+	partition models.Partition,
 	previousError error,
 ) *custom_errors.ExtractorError {
 	lastIdRawValue := lastRow[indexPrimaryKey]
 
 	lastId, ok := convert.ToInt64(lastIdRawValue)
 	if !ok {
-		currentPartition := *partition
+		currentPartition := partition
 		currentPartition.RetryCounter = 3
 		return &custom_errors.ExtractorError{
 			Partition: currentPartition,
@@ -91,7 +92,7 @@ func errorFromLastRow(
 	}
 
 	return &custom_errors.ExtractorError{
-		Partition: *partition,
+		Partition: partition,
 		HasLastId: true,
 		LastId:    lastId,
 		Msg:       previousError.Error(),
@@ -106,21 +107,21 @@ func (mssqlEx *MssqlExtractor) ProcessPartition(
 	partition models.Partition,
 	indexPrimaryKey int,
 	chBatchesOut chan<- models.Batch,
-	rowsRead *int64,
+) (int, error) {
-) error {
+	query := buildExtractQueryMssql(tableInfo, columns, partition.HasRange, partition.Range.IsMinInclusive)
-	query := buildExtractQueryMssql(tableInfo, columns, partition.ShouldUseRange, partition.IsLowerLimitInclusive)
 
 	var queryArgs []any
-	if partition.ShouldUseRange {
+	if partition.HasRange {
 		queryArgs = append(queryArgs,
-			sql.Named("min", partition.LowerLimit),
+			sql.Named("min", partition.Range.Min),
-			sql.Named("max", partition.UpperLimit),
+			sql.Named("max", partition.Range.Max),
 		)
 	}
 
-	rows, err := mssqlEx.db.QueryContext(ctx, query, queryArgs...)
+	rowsRead := 0
+	rows, err := mssqlEx.db.Query(ctx, query, queryArgs...)
 	if err != nil {
-		return &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
 	}
 	defer rows.Close()
 
@@ -136,7 +137,7 @@ func (mssqlEx *MssqlExtractor) ProcessPartition(
 
 		if err := rows.Scan(scanArgs...); err != nil {
 			if len(batchRows) == 0 {
-				return &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+				return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
 			}
 
 			lastRow := batchRows[len(batchRows)-1]
@@ -144,52 +145,48 @@ func (mssqlEx *MssqlExtractor) ProcessPartition(
 			select {
 			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
 			case <-ctx.Done():
-				return nil
+				return rowsRead, ctx.Err()
 			}
 
-			atomic.AddInt64(rowsRead, int64(len(batchRows)))
+			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
-
-			return errorFromLastRow(lastRow, indexPrimaryKey, &partition, err)
 		}
+		rowsRead++
 
 		batchRows = append(batchRows, rowValues)
-
 		if len(batchRows) >= batchSize {
 			select {
 			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
 			case <-ctx.Done():
-				return nil
+				return rowsRead, ctx.Err()
 			}
 
-			atomic.AddInt64(rowsRead, int64(len(batchRows)))
 			batchRows = make([]models.UnknownRowValues, 0, batchSize)
 		}
+
 	}
 
 	if err := rows.Err(); err != nil {
 		if errors.Is(err, ctx.Err()) {
-			return ctx.Err()
+			return rowsRead, ctx.Err()
-		}
-
-		if len(batchRows) == 0 {
-			return &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
 		}
 
+		if len(batchRows) > 0 {
 			lastRow := batchRows[len(batchRows)-1]
-		return errorFromLastRow(lastRow, indexPrimaryKey, &partition, err)
+			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
+		}
+
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
 	}
 
 	if len(batchRows) > 0 {
 		select {
 		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
 		case <-ctx.Done():
-			return nil
+			return rowsRead, ctx.Err()
+		}
 	}
 
-		atomic.AddInt64(rowsRead, int64(len(batchRows)))
+	return rowsRead, nil
-	}
-
-	return nil
 }
 
 func (mssqlEx *MssqlExtractor) Exec(
@@ -234,7 +231,7 @@ func (mssqlEx *MssqlExtractor) Exec(
 				return
 			}
 
-			err := mssqlEx.ProcessPartition(
+			rowsReadResult, err := mssqlEx.ProcessPartition(
 				ctx,
 				tableInfo,
 				columns,
@@ -242,9 +239,12 @@ func (mssqlEx *MssqlExtractor) Exec(
 				partition,
 				indexPrimaryKey,
 				chBatchesOut,
-				rowsRead,
 			)
 
+			if rowsReadResult > 0 {
+				atomic.AddInt64(rowsRead, int64(rowsReadResult))
+			}
+
 			if err != nil {
 				var exError *custom_errors.ExtractorError
 				var jobError *custom_errors.JobError

internal/app/etl/extractors/postgres.go

@@ -6,22 +6,21 @@ import (
 	"fmt"
 	"strings"
 	"sync"
-	"sync/atomic"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	"github.com/google/uuid"
-	"github.com/jackc/pgx/v5/pgxpool"
 )
 
 type PostgresExtractor struct {
-	db *pgxpool.Pool
+	db dbwrapper.DbWrapper
 }
 
-func NewPostgresExtractor(pool *pgxpool.Pool) etl.Extractor {
+func NewPostgresExtractor(db dbwrapper.DbWrapper) etl.Extractor {
-	return &PostgresExtractor{db: pool}
+	return &PostgresExtractor{db: db}
 }
 
 func buildExtractQueryPostgres(sourceDbInfo config.SourceTableInfo, columns []models.ColumnType) string {
@@ -60,17 +59,17 @@ func (postgresEx *PostgresExtractor) ProcessPartition(
 	partition models.Partition,
 	indexPrimaryKey int,
 	chBatchesOut chan<- models.Batch,
-	rowsRead *int64,
+) (int, error) {
-) error {
 	query := buildExtractQueryPostgres(tableInfo, columns)
 
-	if partition.ShouldUseRange {
+	if partition.HasRange {
-		return errors.New("Batch config not yet supported")
+		return 0, errors.New("Batch config not yet supported")
 	}
 
+	rowsRead := 0
 	rows, err := postgresEx.db.Query(ctx, query)
 	if err != nil {
-		return &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
 	}
 	defer rows.Close()
 
@@ -79,8 +78,9 @@ func (postgresEx *PostgresExtractor) ProcessPartition(
 	for rows.Next() {
 		values, err := rows.Values()
 		if err != nil {
-			return errors.New("Unexpected error reading rows from source")
+			return rowsRead, errors.New("Unexpected error reading rows from source")
 		}
+		rowsRead++
 
 		batchRows = append(batchRows, values)
 
@@ -88,29 +88,26 @@ func (postgresEx *PostgresExtractor) ProcessPartition(
 			select {
 			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
 			case <-ctx.Done():
-				return nil
+				return rowsRead, ctx.Err()
 			}
 
-			atomic.AddInt64(rowsRead, int64(len(batchRows)))
 			batchRows = make([]models.UnknownRowValues, 0, batchSize)
 		}
 	}
 
 	if err := rows.Err(); err != nil {
-		return errors.New("Unexpected error reading rows from source")
+		return rowsRead, errors.New("Unexpected error reading rows from source")
 	}
 
 	if len(batchRows) > 0 {
 		select {
 		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
 		case <-ctx.Done():
-			return nil
+			return rowsRead, nil
+		}
 	}
 
-		atomic.AddInt64(rowsRead, int64(len(batchRows)))
+	return rowsRead, nil
-	}
-
-	return nil
 }
 
 func (postgresEx *PostgresExtractor) Exec(

internal/app/etl/extractors/types.go

@@ -1 +0,0 @@
-package extractors

internal/app/etl/loaders/main.go

@@ -9,43 +9,32 @@ import (
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgconn"
-	"github.com/jackc/pgx/v5/pgxpool"
 )
 
-type PostgresLoader struct {
+type GenericLoader struct {
-	db *pgxpool.Pool
+	db dbwrapper.DbWrapper
 }
 
-func NewPostgresLoader(pool *pgxpool.Pool) etl.Loader {
+func NewGenericLoader(db dbwrapper.DbWrapper) etl.Loader {
-	return &PostgresLoader{db: pool}
+	return &GenericLoader{db: db}
 }
 
-func mapSlice[T any, V any](input []T, mapper func(T) V) []V {
+func (gl *GenericLoader) ProcessBatch(
-	result := make([]V, len(input))
-
-	for i, v := range input {
-		result[i] = mapper(v)
-	}
-
-	return result
-}
-
-func (postgresLd *PostgresLoader) ProcessBatch(
 	ctx context.Context,
 	tableInfo config.TargetTableInfo,
 	colNames []string,
 	batch models.Batch,
 ) (int, error) {
-	tableId := pgx.Identifier{tableInfo.Schema, tableInfo.Table}
+	_, err := gl.db.SaveMassive(
-	_, err := postgresLd.db.CopyFrom(
 		ctx,
-		tableId,
+		tableInfo.Schema,
+		tableInfo.Table,
 		colNames,
-		pgx.CopyFromRows(batch.Rows),
+		batch.Rows,
 	)
 
 	if err != nil {
@@ -54,7 +43,7 @@ func (postgresLd *PostgresLoader) ProcessBatch(
 			if pgErr.Code == "23505" {
 				return 0, &custom_errors.JobError{
 					ShouldCancelJob: true,
-					Msg:             fmt.Sprintf("Fatal error in table %s", tableId.Sanitize()),
+					Msg:             fmt.Sprintf("Fatal error in table %s.%s", tableInfo.Schema, tableInfo.Table),
 					Prev:            err,
 				}
 			}
@@ -66,7 +55,7 @@ func (postgresLd *PostgresLoader) ProcessBatch(
 	return len(batch.Rows), nil
 }
 
-func (postgresLd *PostgresLoader) Exec(
+func (gl *GenericLoader) Exec(
 	ctx context.Context,
 	tableInfo config.TargetTableInfo,
 	columns []models.ColumnType,
@@ -93,7 +82,7 @@ func (postgresLd *PostgresLoader) Exec(
 				return
 			}
 
-			processedRows, err := postgresLd.ProcessBatch(ctx, tableInfo, colNames, batch)
+			processedRows, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch)
 
 			if err != nil {
 				var ldError *custom_errors.LoaderError

internal/app/etl/loaders/types.go

@@ -1 +0,0 @@
-package loaders

internal/app/etl/loaders/utils.go

@@ -0,0 +1,11 @@
+package loaders
+
+func mapSlice[T any, V any](input []T, mapper func(T) V) []V {
+	result := make([]V, len(input))
+
+	for i, v := range input {
+		result[i] = mapper(v)
+	}
+
+	return result
+}

internal/app/etl/table_analyzers/main.go

@@ -24,7 +24,7 @@ func PartitionRangeGenerator(
 	if rowsCount <= rowsPerPartition {
 		return []models.Partition{{
 			Id:           uuid.New(),
-			ShouldUseRange: false,
+			HasRange:     false,
 			RetryCounter: 0,
 		}}, nil
 

internal/app/etl/table_analyzers/mssql.go

@@ -8,16 +8,17 @@ import (
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	"github.com/google/uuid"
 )
 
 type MssqlTableAnalyzer struct {
-	db *sql.DB
+	db dbwrapper.DbWrapper
 }
 
-func NewMssqlTableAnalyzer(db *sql.DB) etl.TableAnalyzer {
+func NewMssqlTableAnalyzer(db dbwrapper.DbWrapper) etl.TableAnalyzer {
 	return &MssqlTableAnalyzer{db: db}
 }
 
@@ -35,9 +36,11 @@ JOIN sys.types t ON c.user_type_id = t.user_type_id
 LEFT JOIN sys.types bt ON t.is_user_defined = 1 AND bt.user_type_id = t.system_type_id
 JOIN sys.tables st ON c.object_id = st.object_id
 JOIN sys.schemas s ON st.schema_id = s.schema_id
-WHERE s.name = @schema AND st.name = @table AND c.name NOT LIKE 'graph_id%'
+WHERE s.name = @schema AND st.name = @table AND (c.is_hidden = 0 OR (c.graph_type IS NOT NULL AND c.name LIKE '$%'))
 ORDER BY c.column_id;`
 
+// AND c.name NOT LIKE '$%'
+
 type rawColumnMssql struct {
 	name       string
 	userType   string
@@ -142,7 +145,7 @@ func (ta *MssqlTableAnalyzer) QueryColumnTypes(
 	localCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
 	defer cancel()
 
-	rows, err := ta.db.QueryContext(localCtx, mssqlColumnMetadataQuery, sql.Named("schema", tableInfo.Schema), sql.Named("table", tableInfo.Table))
+	rows, err := ta.db.Query(localCtx, mssqlColumnMetadataQuery, sql.Named("schema", tableInfo.Schema), sql.Named("table", tableInfo.Table))
 	if err != nil {
 		return nil, err
 	}
@@ -183,11 +186,11 @@ JOIN sys.partitions p ON t.object_id = p.object_id
 WHERE s.name = @schema AND t.name = @table AND p.index_id IN (0, 1)
 GROUP BY t.name`
 
-	ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*20)
+	ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Minute)
 	defer cancel()
 
 	var rowsCount int64
-	err := ta.db.QueryRowContext(ctxTimeout, query, sql.Named("schema", tableInfo.Schema), sql.Named("table", tableInfo.Table)).Scan(&rowsCount)
+	err := ta.db.QueryRow(ctxTimeout, query, sql.Named("schema", tableInfo.Schema), sql.Named("table", tableInfo.Table)).Scan(&rowsCount)
 	if err != nil {
 		return 0, err
 	}
@@ -215,10 +218,10 @@ ORDER BY batch_id`,
 		tableInfo.Schema,
 		tableInfo.Table)
 
-	ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*20)
+	ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Minute)
 	defer cancel()
 
-	rows, err := ta.db.QueryContext(ctxTimeout, query, sql.Named("maxPartitions", maxPartitions))
+	rows, err := ta.db.Query(ctxTimeout, query, sql.Named("maxPartitions", maxPartitions))
 	if err != nil {
 		return nil, err
 	}
@@ -229,12 +232,14 @@ ORDER BY batch_id`,
 	for rows.Next() {
 		partition := models.Partition{
 			Id:           uuid.New(),
-			ShouldUseRange:        true,
+			HasRange:     true,
 			RetryCounter: 0,
-			IsLowerLimitInclusive: true,
+			Range: models.PartitionRange{
+				IsMinInclusive: true,
+			},
 		}
 
-		if err := rows.Scan(&partition.LowerLimit, &partition.UpperLimit); err != nil {
+		if err := rows.Scan(&partition.Range.Min, &partition.Range.Max); err != nil {
 			return nil, err
 		}
 

internal/app/etl/table_analyzers/postgres.go

@@ -6,16 +6,16 @@ import (
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/jackc/pgx/v5/pgxpool"
 )
 
 type PostgresTableAnalyzer struct {
-	db *pgxpool.Pool
+	db dbwrapper.DbWrapper
 }
 
-func NewPostgresTableAnalyzer(db *pgxpool.Pool) etl.TableAnalyzer {
+func NewPostgresTableAnalyzer(db dbwrapper.DbWrapper) etl.TableAnalyzer {
 	return &PostgresTableAnalyzer{db: db}
 }
 
@@ -125,7 +125,7 @@ func (ta *PostgresTableAnalyzer) QueryColumnTypes(
 	ctx context.Context,
 	tableInfo config.TableInfo,
 ) ([]models.ColumnType, error) {
-	localCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	localCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
 	defer cancel()
 
 	rows, err := ta.db.Query(localCtx, postgresColumnMetadataQuery, tableInfo.Schema, tableInfo.Table)

internal/app/etl/transformers/mssql.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"sync"
-	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
@@ -18,46 +17,7 @@ func NewMssqlTransformer() etl.Transformer {
 }
 
 func computeTransformationPlan(columns []models.ColumnType) []etl.ColumnTransformPlan {
-	var plan []etl.ColumnTransformPlan
+	return []etl.ColumnTransformPlan{}
-
-	for i, col := range columns {
-		switch col.SystemType() {
-		case "uniqueidentifier":
-			plan = append(plan, etl.ColumnTransformPlan{
-				Index: i,
-				Fn: func(v any) (any, error) {
-					if b, ok := v.([]byte); ok && b != nil {
-						return mssqlUuidToBigEndian(b)
-					}
-					return v, nil
-				},
-			})
-
-		case "geometry", "geography":
-			plan = append(plan, etl.ColumnTransformPlan{
-				Index: i,
-				Fn: func(v any) (any, error) {
-					if b, ok := v.([]byte); ok && b != nil {
-						return wkbToEwkbWithSrid(b, 4326)
-					}
-					return v, nil
-				},
-			})
-
-		case "datetime", "datetime2":
-			plan = append(plan, etl.ColumnTransformPlan{
-				Index: i,
-				Fn: func(v any) (any, error) {
-					if t, ok := v.(time.Time); ok {
-						return ensureUTC(t), nil
-					}
-					return v, nil
-				},
-			})
-		}
-	}
-
-	return plan
 }
 
 const processBatchCtxCheck = 4096

internal/app/etl/types.go

@@ -18,8 +18,7 @@ type Extractor interface {
 		partition models.Partition,
 		indexPrimaryKey int,
 		chBatchesOut chan<- models.Batch,
-		rowsRead *int64,
+	) (int, error)
-	) error
 
 	Exec(
 		ctx context.Context,

internal/app/models/main.go

@@ -1,6 +1,10 @@
 package models
 
-import "github.com/google/uuid"
+import (
+	"time"
+
+	"github.com/google/uuid"
+)
 
 type UnknownRowValues = []any
 
@@ -11,12 +15,27 @@ type Batch struct {
 	RetryCounter int
 }
 
+type PartitionRange struct {
+	Min            int64
+	Max            int64
+	IsMinInclusive bool
+	IsMaxInclusive bool
+}
+
 type Partition struct {
 	Id           uuid.UUID
 	ParentId     uuid.UUID
-	LowerLimit            int64
+	Range        PartitionRange
-	UpperLimit            int64
+	HasRange     bool
-	IsLowerLimitInclusive bool
-	ShouldUseRange        bool
 	RetryCounter int
 }
+
+type JobResult struct {
+	JobName    string
+	StartTime  time.Time
+	Duration   time.Duration
+	RowsRead   int64
+	RowsLoaded int64
+	RowsFailed int64
+	Error      error
+}

scripts/pg-info-test/main.go

@@ -8,13 +8,32 @@ import (
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/jackc/pgx/v5/pgxpool"
 	log "github.com/sirupsen/logrus"
 )
 
+func Connect(ctx context.Context, dbURL string) (*pgxpool.Pool, error) {
+	pool, err := pgxpool.New(ctx, dbURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to connect to database: %w", err)
+	}
+
+	if err := pool.Ping(ctx); err != nil {
+		pool.Close()
+		return nil, fmt.Errorf("unable to ping database: %w", err)
+	}
+
+	return pool, nil
+}
+
+func Close(pool *pgxpool.Pool) {
+	if pool != nil {
+		pool.Close()
+	}
+}
+
 func main() {
 	log.SetFormatter(&log.TextFormatter{
 		FullTimestamp:   true,
@@ -27,8 +46,8 @@ func main() {
 	ctxSource, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 
-	sourcePool, err := db.Connect(ctxSource, config.App.SourceDbUrl)
+	sourcePool, err := Connect(ctxSource, config.App.SourceDbUrl)
-	defer db.Close(sourcePool)
+	defer Close(sourcePool)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -37,8 +56,8 @@ func main() {
 	ctxTarget, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 
-	targetPool, err := db.Connect(ctxTarget, config.App.TargetDbUrl)
+	targetPool, err := Connect(ctxTarget, config.App.TargetDbUrl)
-	defer db.Close(targetPool)
+	defer Close(targetPool)
 	if err != nil {
 		log.Fatal(err)
 	}