feat: add backoff package for retry logic and implement example usage in new backoff-test script

.atl/skill-registry.md

@@ -1,24 +0,0 @@
-# Skill Registry — go-migrate
-
-Generated: 2026-04-21
-
-## Compact Rules
-
-### Go conventions
-- Use existing error wrapping pattern: `fmt.Errorf("context: %w", err)`
-- Channel-based pipeline — keep goroutine lifecycle clean (close channels in correct order)
-- No comments unless non-obvious WHY; no docstrings
-- Prefer named returns only when it aids clarity in short functions
-- Use `strings.EqualFold` for case-insensitive column name comparison
-
-### Project conventions
-- Config structs live in `internal/app/config/`
-- ETL interfaces live in `internal/app/etl/types.go`
-- Transformer implementations in `internal/app/etl/transformers/`
-- Azure operations via `internal/app/azure/main.go`
-- Per-job transformer creation (not shared) when job has storage config
-
-## User Skills
-| Trigger | Skill |
-|---------|-------|
-| sdd-* | SDD workflow skills |

.env.example

@@ -1,12 +1,2 @@
-SOURCE_DB_URL=sqlserver://sa:password@localhost:1433?database=master&packet+size=32767&loc=UTC
-TARGET_DB_URL=postgresql://postgres:password@localhost:5432/db
-
-LOG_LEVEL=INFO
-
-AZ_STORAGE_ENABLED=false
-AZ_ACCOUNT_NAME=
-AZ_CONTAINER=
-AZ_ACCOUNT_KEY=
-AZ_USE_HTTPS=true
-AZ_SERVICE_URL=
-AZ_PREFIX=
+PG_FROM_DB_URL=postgresql://postgres:password@localhost:5432/db
+PG_TO_DB_URL=postgresql://postgres:password@localhost:5432/db

.gitignore

@@ -27,5 +27,5 @@ go.work.sum
 
 # Editor/IDE
 # .idea/
-.vscode/
+# .vscode/
 .temp

cmd/go_migrate/log.go

@@ -3,7 +3,6 @@ package main
 import (
 	"time"
 
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -14,13 +13,5 @@ func configureLog() {
 		DisableSorting:  false,
 		PadLevelText:    true,
 	})
-
-	logLevelEnv := config.App.LogLevel
-	logLevel, err := log.ParseLevel(logLevelEnv)
-	if err != nil {
-		log.Warnf("Nivel de log inválido '%s', usando INFO por defecto", logLevelEnv)
-		logLevel = log.InfoLevel
-	}
-
-	log.SetLevel(logLevel)
+	log.SetLevel(log.DebugLevel)
 }

cmd/go_migrate/main.go

@@ -5,12 +5,12 @@ import (
 	"sync"
 	"time"
 
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/azure"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractors"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loaders"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
@@ -24,7 +24,7 @@ func main() {
 		log.Fatalf("error leyendo configuracion: %v", err)
 	}
 
-	// log.Debugf("Config: %+v", migrationConfig)
+	log.Debugf("Config: %+v", migrationConfig)
 
 	startTime := time.Now()
 
@@ -118,17 +118,9 @@ func processMigrationJobs(
 
 	sourceTableAnalyzer := table_analyzers.NewMssqlTableAnalyzer(sourceDb)
 	targetTableAnalyzer := table_analyzers.NewPostgresTableAnalyzer(targetDb)
-	extractor := extractors.NewExtractor(sourceDb)
-	loader := loaders.NewGenericLoader(targetDb)
-
-	var azureClient *azure.Client
-	if config.App.AzureStorage.Enabled {
-		var err error
-		azureClient, err = azure.NewClient(config.App.AzureStorage)
-		if err != nil {
-			log.Fatalf("Failed to create Azure storage client: %v", err)
-		}
-	}
+	extractor := extractors.NewMssqlExtractor(sourceDb)
+	transformer := transformers.NewMssqlTransformer()
+	loader := loaders.NewPostgresLoader(targetDb)
 
 	for i := range maxParallelWorkers {
 		wgJobs.Go(func() {
@@ -140,10 +132,9 @@ func processMigrationJobs(
 					sourceTableAnalyzer,
 					targetTableAnalyzer,
 					extractor,
-					azureClient,
+					transformer,
 					loader,
 					job,
-					targetDb.GetDialect(),
 				)
 
 				chJobResults <- res

cmd/go_migrate/process.go

@@ -7,48 +7,26 @@ import (
 	"sync/atomic"
 	"time"
 
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/azure"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
 	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loaders"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 )
 
-const jobErrorsChannelSize int = 100
-
-func buildTruncateQuery(targetDbType, schema, table, truncateMethod string) string {
-	if truncateMethod == "DELETE" {
-		if targetDbType == "postgres" {
-			return fmt.Sprintf(`DELETE FROM "%s"."%s"`, schema, table)
-		}
-		return fmt.Sprintf(`DELETE FROM [%s].[%s]`, schema, table)
-	}
-
-	if targetDbType == "postgres" {
-		return fmt.Sprintf(`TRUNCATE TABLE "%s"."%s"`, schema, table)
-	}
-	return fmt.Sprintf(`TRUNCATE TABLE [%s].[%s]`, schema, table)
-}
-
 func processMigrationJob(
 	ctx context.Context,
 	targetDbWrapper dbwrapper.DbWrapper,
 	sourceTableAnalyzer etl.TableAnalyzer,
 	targetTableAnalyzer etl.TableAnalyzer,
-	extractor extractors.GenericExtractor,
-	azureClient *azure.Client,
-	loader loaders.GenericLoader,
+	extractor etl.Extractor,
+	transformer etl.Transformer,
+	loader etl.Loader,
 	job config.Job,
-	targetDbType string,
 ) models.JobResult {
-	transformer := transformers.NewMssqlTransformer(job.ToStorage, job.SourceTable, azureClient)
 	localCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -88,13 +66,7 @@ func processMigrationJob(
 		return result
 	}
 
-	preSqlQueries := job.TargetTable.PreSQL
-	if job.TruncateTarget {
-		truncateQuery := buildTruncateQuery(targetDbType, job.TargetTable.Schema, job.TargetTable.Table, job.TruncateMethod)
-		preSqlQueries = append([]string{truncateQuery}, job.TargetTable.PreSQL...)
-	}
-
-	for _, query := range preSqlQueries {
+	for _, query := range job.PreSQL {
 		if _, err := targetDbWrapper.Exec(localCtx, query); err != nil {
 			result.Error = err
 			return result
@@ -107,17 +79,17 @@ func processMigrationJob(
 		job.SourceTable.TableInfo,
 		job.SourceTable.PrimaryKey,
 		job.RowsPerPartition,
-		job.Range,
 	)
 	if err != nil {
 		log.Error("Unexpected error calculating batch ranges: ", err)
 	}
 
-	chJobErrors := make(chan custom_errors.JobError, jobErrorsChannelSize)
-	chLoadersErrors := make(chan custom_errors.LoaderError, job.ExtractorQueueSize)
-	chPartitions := make(chan models.Partition, job.ExtractorQueueSize)
-	chBatchesRaw := make(chan models.Batch, job.ExtractorQueueSize)
-	chBatchesTransformed := make(chan models.Batch, job.TransformerQueueSize)
+	chJobErrors := make(chan custom_errors.JobError, job.QueueSize)
+	chExtractorErrors := make(chan custom_errors.ExtractorError, job.QueueSize)
+	chLoadersErrors := make(chan custom_errors.LoaderError, job.QueueSize)
+	chPartitions := make(chan models.Partition, job.QueueSize)
+	chBatchesRaw := make(chan models.Batch, job.QueueSize)
+	chBatchesTransformed := make(chan models.Batch, job.QueueSize)
 
 	var wgActivePartitions sync.WaitGroup
 	var wgActiveBatches sync.WaitGroup
@@ -133,10 +105,19 @@ func processMigrationJob(
 		}
 	}()
 
+	go custom_errors.ExtractorErrorHandler(
+		localCtx,
+		job.Retry,
+		job.MaxPartitionErrrors,
+		chExtractorErrors,
+		chPartitions,
+		chJobErrors,
+		&wgActivePartitions,
+	)
 	go custom_errors.LoaderErrorHandler(
 		localCtx,
 		job.Retry,
-		job.MaxExtractorBatchErrors,
+		job.MaxChunkErrors,
 		chLoadersErrors,
 		chBatchesTransformed,
 		chJobErrors,
@@ -148,18 +129,17 @@ func processMigrationJob(
 
 	for range maxExtractors {
 		wgExtractors.Go(func() {
-			extractor.Consume(
+			extractor.Exec(
 				localCtx,
 				job.SourceTable,
 				sourceColTypes,
-				job.ExtractorBatchSize,
-				job.Retry,
+				job.BatchSize,
 				chPartitions,
 				chBatchesRaw,
+				chExtractorErrors,
 				chJobErrors,
 				&wgActivePartitions,
 				&rowsRead,
-				job.SourceTable.FromJsonColumns,
 			)
 		})
 	}
@@ -190,12 +170,12 @@ func processMigrationJob(
 
 	for range job.MaxLoaders {
 		wgLoaders.Go(func() {
-			loader.Consume(
+			loader.Exec(
 				localCtx,
 				job.TargetTable,
 				targetColTypes,
-				job.Retry,
 				chBatchesTransformed,
+				chLoadersErrors,
 				chJobErrors,
 				&wgActiveBatches,
 				&rowsLoaded,
@@ -210,6 +190,8 @@ func processMigrationJob(
 		log.Debugf("wgActivePartitions is empty (%v)", job.Name)
 		close(chPartitions)
 		log.Debugf("chPartitions is closed (%v)", job.Name)
+		close(chExtractorErrors)
+		log.Debugf("chExtractorErrors is closed (%v)", job.Name)
 
 		wgExtractors.Wait()
 		log.Debugf("wgExtractors is empty (%v)", job.Name)
@@ -232,7 +214,7 @@ func processMigrationJob(
 		cancel()
 	}()
 
-	for _, query := range job.TargetTable.PostSQL {
+	for _, query := range job.PostSQL {
 		if _, err := targetDbWrapper.Exec(localCtx, query); err != nil {
 			result.Error = err
 			return result

config.yaml

@@ -3,19 +3,15 @@ source_db_type: sqlserver
 target_db_type: postgres
 
 defaults:
-  batches_per_partition: 8
   max_extractors: 2
-  extractor_batch_size: 25000
-  extractor_queue_size: 8
-  max_transformers: 2
-  transformer_batch_size: 25000
-  transformer_queue_size: 8
   max_loaders: 4
-  loader_batch_size: 25000
+  queue_size: 8
+  batch_size: 25000
+  batches_per_partition: 8
   truncate_target: true
   truncate_method: TRUNCATE # TRUNCATE | DELETE
   max_partition_errrors: 5
-  max_extractor_batch_errors: 5
+  max_chunk_errors: 5
   retry:
     attempts: 3
     base_delay_ms: 500
@@ -34,6 +30,7 @@ jobs:
       table: MANZANA
     pre_sql: 
       - 'SELECT 1'
+      # - 'TRUNCATE TABLE "Cartografia"."MANZANA"'
     range:
       min: 1000000
       max: 2000000
@@ -46,36 +43,11 @@ jobs:
       schema: Red
       table: PUERTO
       primary_key: ID_PUERTO
-      from_json:
-        - column: $node_id*
-          field: id
     target:
       schema: Red
       table: PUERTO
     pre_sql: 
       - 'SELECT 1'
+      # - 'TRUNCATE TABLE "Red"."PUERTO"'
     post_sql: 
       - "SELECT 1"
-
-  - name: infraestructura_site_holder__attach
-    source:
-      schema: Infraestructura
-      table: SITE_HOLDER__ATTACH
-      primary_key: GDB_ARCHIVE_OID
-    target:
-      schema: Infraestructura
-      table: SITE_HOLDER__ATTACH
-    to_storage:
-      columns:
-        - source: DATA
-          target: FILE_URL
-          mode: REFERENCE_ONLY
-    max_extractors: 8
-    max_loaders: 4
-    queue_size: 32
-    batch_size: 1
-    retry:
-      attempts: 5
-      base_delay_ms: 1000
-      max_delay_ms: 15000
-      max_jitter_ms: 500

go.mod

@@ -1,13 +1,12 @@
 module git.ksdemosapps.com/kylesoda/go-migrate
 
-go 1.26
+go 1.25.7
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
 	github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4
 	github.com/google/uuid v1.6.0
-	github.com/ilyakaznacheev/cleanenv v1.5.0
 	github.com/jackc/pgx/v5 v5.9.1
+	github.com/joho/godotenv v1.5.1
 	github.com/microsoft/go-mssqldb v1.9.8
 	github.com/sirupsen/logrus v1.9.4
 	github.com/twpayne/go-geom v1.6.1
@@ -16,20 +15,17 @@ require (
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/BurntSushi/toml v1.6.0 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
-	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
-	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
 )

go.sum

@@ -4,25 +4,23 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpz
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 h1:E4MgwLBGeVB5f2MdcIVD3ELVAWpr+WD6MUe1i+tM/PA=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
-github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
-github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
 github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
 github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
 github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -38,8 +36,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4=
-github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -50,8 +46,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
@@ -91,5 +87,3 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ=
-olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw=

internal/app/azure/main.go

@@ -1,87 +0,0 @@
-package azure
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net/url"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-)
-
-var (
-	ErrInvalidConnectionString = errors.New("invalid connection string")
-	ErrContainerNotFound       = errors.New("container not found")
-	ErrBlobNotFound            = errors.New("blob not found")
-	ErrInvalidInput            = errors.New("invalid input parameters")
-)
-
-type Client struct {
-	client             *azblob.Client
-	azureStorageConfig config.AzureStorageConfig
-}
-
-func NewClient(azureStorageConfig config.AzureStorageConfig) (*Client, error) {
-	protocol := "https"
-	if !azureStorageConfig.UseHTTPS {
-		protocol = "http"
-	}
-
-	blobEndpoint, _ := url.JoinPath(azureStorageConfig.ServiceURL, azureStorageConfig.AccountName)
-	connStr := fmt.Sprintf("DefaultEndpointsProtocol=%s;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;",
-		protocol, azureStorageConfig.AccountName, azureStorageConfig.AccountKey, blobEndpoint)
-
-	client, err := azblob.NewClientFromConnectionString(connStr, nil)
-	if err != nil {
-		return nil, fmt.Errorf("creating azure storage client: %w", err)
-	}
-
-	return &Client{
-		client:             client,
-		azureStorageConfig: azureStorageConfig,
-	}, nil
-}
-
-func (c *Client) CreateContainer(ctx context.Context, containerName string) error {
-	if containerName == "" {
-		return ErrInvalidInput
-	}
-
-	_, err := c.client.CreateContainer(ctx, containerName, nil)
-	if err != nil {
-		return fmt.Errorf("creating container %s: %w", containerName, err)
-	}
-	return nil
-}
-
-func (c *Client) UploadBuffer(ctx context.Context, containerName, blobPath string, buffer []byte) error {
-	if containerName == "" || blobPath == "" || buffer == nil {
-		return ErrInvalidInput
-	}
-
-	_, err := c.client.UploadBuffer(ctx, containerName, blobPath, buffer, nil)
-	if err != nil {
-		return fmt.Errorf("uploading blob %s: %w", blobPath, err)
-	}
-	return nil
-}
-
-func (c *Client) UploadAndGetURL(ctx context.Context, blobPath string, buffer []byte) (string, error) {
-	if blobPath == "" || buffer == nil {
-		return "", ErrInvalidInput
-	}
-
-	fullPath := blobPath
-	if c.azureStorageConfig.Prefix != "" {
-		fullPath, _ = url.JoinPath(c.azureStorageConfig.Prefix, blobPath)
-	}
-
-	if err := c.UploadBuffer(ctx, c.azureStorageConfig.Container, fullPath, buffer); err != nil {
-		return "", err
-	}
-
-	blobEndpoint, _ := url.JoinPath(c.azureStorageConfig.ServiceURL, c.azureStorageConfig.AccountName)
-	blobURL, _ := url.JoinPath(blobEndpoint, c.azureStorageConfig.Container, fullPath)
-	return blobURL, nil
-}

internal/app/config/main.go

@@ -1,41 +1,41 @@
 package config
 
 import (
-	"github.com/ilyakaznacheev/cleanenv"
+	"os"
+
+	"github.com/joho/godotenv"
 	log "github.com/sirupsen/logrus"
 )
 
-type AzureStorageConfig struct {
-	AccountName string `env:"AZ_ACCOUNT_NAME"`
-	Container   string `env:"AZ_CONTAINER"`
-	AccountKey  string `env:"AZ_ACCOUNT_KEY"`
-	UseHTTPS    bool   `env:"AZ_USE_HTTPS" env-default:"true"`
-	ServiceURL  string `env:"AZ_SERVICE_URL"`
-	Prefix      string `env:"AZ_PREFIX"`
-	Enabled     bool   `env:"AZ_STORAGE_ENABLED"`
+type appConfig struct {
+	SourceDbUrl string
+	TargetDbUrl string
 }
 
-type appConfig struct {
-	SourceDbUrl  string `env:"SOURCE_DB_URL" env-required:"true"`
-	TargetDbUrl  string `env:"TARGET_DB_URL" env-required:"true"`
-	LogLevel     string `env:"LOG_LEVEL" env-default:"INFO"`
-	AzureStorage AzureStorageConfig
+func loadEnv() {
+	err := godotenv.Load()
+	if err != nil {
+		log.Warn("Warning: could not load .env file")
+	}
 }
 
 func getAppConfig() appConfig {
-	var cfg appConfig
+	loadEnv()
 
-	err := cleanenv.ReadConfig(".env", &cfg)
-	if err != nil {
-		log.Warn("Could not load .env file")
+	sourceDbUrl := os.Getenv("SOURCE_DB_URL")
+	if sourceDbUrl == "" {
+		log.Fatal("SOURCE_DB_URL environment variable not set")
 	}
 
-	err = cleanenv.ReadEnv(&cfg)
-	if err != nil {
-		log.Fatalf("Error al cargar variables: %v", err)
+	targetDbUrl := os.Getenv("TARGET_DB_URL")
+	if targetDbUrl == "" {
+		log.Fatal("TARGET_DB_URL environment variable not set")
 	}
 
-	return cfg
+	return appConfig{
+		SourceDbUrl: sourceDbUrl,
+		TargetDbUrl: targetDbUrl,
+	}
 }
 
 var App appConfig = getAppConfig()

internal/app/config/migration.go

@@ -14,38 +14,18 @@ type RetryConfig struct {
 	MaxJitterMs int `yaml:"max_jitter_ms"`
 }
 
-type ToStorageColumnConfig struct {
-	Source string `yaml:"source"`
-	Target string `yaml:"target"`
-	Mode   string `yaml:"mode"`
-}
-
-type ToStorageConfig struct {
-	Columns []ToStorageColumnConfig `yaml:"columns"`
-}
-
 type JobConfig struct {
-	BatchesPerPartition     int         `yaml:"batches_per_partition"`
 	MaxExtractors       int         `yaml:"max_extractors"`
-	ExtractorBatchSize      int         `yaml:"extractor_batch_size"`
-	ExtractorQueueSize      int         `yaml:"extractor_queue_size"`
-	MaxTransformers         int         `yaml:"max_transformers"`
-	TransformerBatchSize    int         `yaml:"transformer_batch_size"`
-	TransformerQueueSize    int         `yaml:"transformer_queue_size"`
 	MaxLoaders          int         `yaml:"max_loaders"`
-	LoaderBatchSize         int         `yaml:"loader_batch_size"`
+	QueueSize           int         `yaml:"queue_size"`
+	BatchSize           int         `yaml:"batch_size"`
+	BatchesPerPartition int         `yaml:"batches_per_partition"`
 	TruncateTarget      bool        `yaml:"truncate_target"`
 	TruncateMethod      string      `yaml:"truncate_method"`
 	MaxPartitionErrrors int         `yaml:"max_partition_errrors"`
-	MaxExtractorBatchErrors int         `yaml:"max_extractor_batch_errors"`
+	MaxChunkErrors      int         `yaml:"max_chunk_errors"`
 	Retry               RetryConfig `yaml:"retry"`
 	RowsPerPartition    int64
-	ToStorage               ToStorageConfig `yaml:"to_storage"`
-}
-
-type FromJsonItem struct {
-	Column string `yaml:"column"`
-	Field  string `yaml:"field"`
 }
 
 type TableInfo struct {
@@ -53,23 +33,13 @@ type TableInfo struct {
 	Table  string `yaml:"table"`
 }
 
+type TargetTableInfo struct {
+	TableInfo `yaml:",inline"`
+}
+
 type SourceTableInfo struct {
 	TableInfo  `yaml:",inline"`
 	PrimaryKey string `yaml:"primary_key"`
-	FromJsonColumns []FromJsonItem `yaml:"from_json"`
-}
-
-type TargetTableInfo struct {
-	TableInfo `yaml:",inline"`
-	PreSQL    []string `yaml:"pre_sql"`
-	PostSQL   []string `yaml:"post_sql"`
-}
-
-type RangeConfig struct {
-	Min            int64 `yaml:"min"`
-	Max            int64 `yaml:"max"`
-	IsMinInclusive bool  `yaml:"is_min_inclusive"`
-	IsMaxInclusive bool  `yaml:"is_max_inclusive"`
 }
 
 type Job struct {
@@ -77,8 +47,15 @@ type Job struct {
 	Enabled     bool            `yaml:"enabled"`
 	SourceTable SourceTableInfo `yaml:"source"`
 	TargetTable TargetTableInfo `yaml:"target"`
+	PreSQL      []string        `yaml:"pre_sql"`
+	PostSQL     []string        `yaml:"post_sql"`
 	JobConfig   `yaml:",inline"`
-	Range       RangeConfig `yaml:"range"`
+	Range       struct {
+		Min            int64 `yaml:"min"`
+		Max            int64 `yaml:"max"`
+		IsMinInclusive bool  `yaml:"is_min_inclusive"`
+		IsMaxInclusive bool  `yaml:"is_max_inclusive"`
+	}
 }
 
 type MigrationConfig struct {
@@ -107,7 +84,7 @@ func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
 	c.Defaults = raw.Defaults
 	c.SourceDbType = raw.SourceDbType
 	c.TargetDbType = raw.TargetDbType
-	c.Defaults.RowsPerPartition = int64(raw.Defaults.ExtractorBatchSize * raw.Defaults.BatchesPerPartition)
+	c.Defaults.RowsPerPartition = int64(raw.Defaults.BatchSize * raw.Defaults.BatchesPerPartition)
 
 	for _, node := range raw.Jobs {
 		job := Job{
@@ -118,7 +95,7 @@ func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
 			return err
 		}
 
-		job.RowsPerPartition = int64(job.ExtractorBatchSize * job.BatchesPerPartition)
+		job.RowsPerPartition = int64(job.BatchSize * job.BatchesPerPartition)
 
 		c.Jobs = append(c.Jobs, job)
 	}

internal/app/custom_errors/backoff.go

@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-func ComputeBackoffDelay(retryCounter int, baseDelayMs int, maxDelayMs int, maxJitterMs int) time.Duration {
+func computeBackoffDelay(retryCounter int, baseDelayMs int, maxDelayMs int, maxJitterMs int) time.Duration {
 	if retryCounter < 0 {
 		retryCounter = 0
 	}

internal/app/custom_errors/extractor.error.go

@@ -1,7 +1,13 @@
 package custom_errors
 
 import (
+	"context"
+	"fmt"
+	"sync"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/google/uuid"
 )
 
 type ExtractorError struct {
@@ -14,3 +20,100 @@ type ExtractorError struct {
 func (e *ExtractorError) Error() string {
 	return e.Msg
 }
+
+func ExtractorErrorHandler(
+	ctx context.Context,
+	retryConfig config.RetryConfig,
+	maxPartitionErrors int,
+	chErrorsIn <-chan ExtractorError,
+	chPartitionsOut chan<- models.Partition,
+	chJobErrorsOut chan<- JobError,
+	wgActivePartitions *sync.WaitGroup,
+) {
+	definitiveErrors := 0
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+
+		case err, ok := <-chErrorsIn:
+			if !ok {
+				return
+			}
+
+			if err.Partition.RetryCounter >= retryConfig.Attempts {
+				wgActivePartitions.Done()
+				definitiveErrors++
+				jobError := JobError{
+					ShouldCancelJob: false,
+					Msg:             fmt.Sprintf("Partition %v reached max retries (%d)", err.Partition.Id, retryConfig.Attempts),
+					Prev:            &err,
+				}
+
+				select {
+				case chJobErrorsOut <- jobError:
+				case <-ctx.Done():
+					return
+				}
+
+				if maxPartitionErrors > 0 && definitiveErrors >= maxPartitionErrors {
+					fatalError := JobError{
+						ShouldCancelJob: true,
+						Msg:             fmt.Sprintf("Partition error limit reached (%d)", maxPartitionErrors),
+						Prev:            &err,
+					}
+
+					select {
+					case chJobErrorsOut <- fatalError:
+					case <-ctx.Done():
+						return
+					}
+				}
+
+				continue
+			} else {
+				jobError := JobError{
+					ShouldCancelJob: false,
+					Msg:             fmt.Sprintf("Temporal error in partition %v (retries: %d)", err.Partition.Id, err.Partition.RetryCounter),
+					Prev:            &err,
+				}
+
+				select {
+				case chJobErrorsOut <- jobError:
+				case <-ctx.Done():
+					return
+				}
+			}
+
+			newPartition := err.Partition
+			newPartition.RetryCounter++
+
+			delay := computeBackoffDelay(
+				newPartition.RetryCounter,
+				retryConfig.BaseDelayMs,
+				retryConfig.MaxDelayMs,
+				retryConfig.MaxJitterMs,
+			)
+
+			if err.HasLastId {
+				newPartition.ParentId = err.Partition.Id
+				newPartition.Id = uuid.New()
+				newPartition.Range.Min = err.LastId
+				newPartition.Range.IsMinInclusive = false
+			}
+
+			requeueWithBackoff(ctx, delay, func() {
+				select {
+				case chPartitionsOut <- newPartition:
+				case <-ctx.Done():
+					return
+				}
+			})
+		}
+	}
+}

internal/app/custom_errors/loader.error.go

@@ -88,7 +88,7 @@ func LoaderErrorHandler(
 			}
 
 			err.Batch.RetryCounter++
-			delay := ComputeBackoffDelay(
+			delay := computeBackoffDelay(
 				err.Batch.RetryCounter,
 				retryConfig.BaseDelayMs,
 				retryConfig.MaxDelayMs,

internal/app/db-wrapper/db_dialects/main.go

@@ -1,7 +0,0 @@
-package db_dialects
-
-const (
-	SqlServer string = "sqlserver"
-	Postgres  string = "postgres"
-	Null      string = "null"
-)

internal/app/db-wrapper/mssql.go

@@ -4,18 +4,13 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"strings"
 
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	dbdialects "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper/db_dialects"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	mssql "github.com/microsoft/go-mssqldb"
-	"github.com/sirupsen/logrus"
 )
 
 func init() {
-	Register(dbdialects.SqlServer, func() DbWrapper {
-		return &mssqlDbWrapper{dialect: dbdialects.SqlServer}
+	Register("sqlserver", func() DbWrapper {
+		return &mssqlDbWrapper{dialect: "sqlserver"}
 	})
 }
 
@@ -179,142 +174,3 @@ func (mw *mssqlDbWrapper) SaveMassive(ctx context.Context, schema string, table
 
 	return rowsAffected, nil
 }
-
-func buildExtractQueryMssql(q ExtractionQuery) (string, error) {
-	var sbQuery strings.Builder
-
-	sbQuery.WriteString("SELECT ")
-
-	hasRegularColumns := len(q.Columns) > 0
-	hasJsonColumns := len(q.FromJsonColumns) > 0
-
-	// logrus.Debugf("Extraction query: %+v", q)
-
-	resolvedJson := make(map[string][]config.FromJsonItem, len(q.FromJsonColumns))
-	if hasJsonColumns {
-		for _, jsonConfig := range q.FromJsonColumns {
-			actualColumnName, err := findColumnByPattern(q.Columns, jsonConfig.Column)
-			if err != nil {
-				return "", err
-			}
-			resolvedJson[actualColumnName] = append(resolvedJson[actualColumnName], jsonConfig)
-		}
-	}
-
-	selectParts := make([]string, 0, len(q.Columns)+len(q.FromJsonColumns))
-	if hasRegularColumns {
-		for _, col := range q.Columns {
-			jsonConfigs, isJsonColumn := resolvedJson[col.Name()]
-			if isJsonColumn {
-				for _, jsonConfig := range jsonConfigs {
-					jsonPath := buildJsonPathMssql(jsonConfig.Field)
-					jsonExpr := fmt.Sprintf("JSON_VALUE([%s], '%s') AS [%s]", col.Name(), jsonPath, col.Name())
-					selectParts = append(selectParts, jsonExpr)
-				}
-				continue
-			}
-
-			colExpr := fmt.Sprintf("[%s]", col.Name())
-			switch col.Type() {
-			case "GEOMETRY":
-				colExpr = fmt.Sprintf("[%s].STAsBinary() AS [%s]", col.Name(), col.Name())
-			}
-			selectParts = append(selectParts, colExpr)
-		}
-	} else if !hasJsonColumns {
-		selectParts = append(selectParts, "*")
-	}
-
-	for i, part := range selectParts {
-		sbQuery.WriteString(part)
-		if i < len(selectParts)-1 {
-			sbQuery.WriteString(", ")
-		}
-	}
-
-	fmt.Fprintf(&sbQuery, " FROM [%s].[%s]", q.Schema, q.Table)
-
-	if q.LowerLimit.IsValid || q.UpperLimit.IsValid {
-		sbQuery.WriteString(" WHERE ")
-
-		if q.LowerLimit.IsValid {
-			fmt.Fprintf(&sbQuery, "[%s]", q.PrimaryKey)
-			if q.LowerLimit.IsInclusive {
-				sbQuery.WriteString(" >=")
-			} else {
-				sbQuery.WriteString(" >")
-			}
-			sbQuery.WriteString(" @min")
-		}
-
-		if q.LowerLimit.IsValid && q.UpperLimit.IsValid {
-			sbQuery.WriteString(" AND ")
-		}
-
-		if q.UpperLimit.IsValid {
-			fmt.Fprintf(&sbQuery, "[%s]", q.PrimaryKey)
-			if q.UpperLimit.IsInclusive {
-				sbQuery.WriteString(" <=")
-			} else {
-				sbQuery.WriteString(" <")
-			}
-			sbQuery.WriteString(" @max")
-		}
-	}
-
-	fmt.Fprintf(&sbQuery, " ORDER BY [%s] ASC", q.PrimaryKey)
-
-	return sbQuery.String(), nil
-}
-
-func findColumnByPattern(columns []models.ColumnType, pattern string) (string, error) {
-	if pattern == "" {
-		return "", fmt.Errorf("column pattern cannot be empty")
-	}
-
-	if before, ok := strings.CutSuffix(pattern, "*"); ok {
-		prefix := before
-		for _, col := range columns {
-			if strings.HasPrefix(col.Name(), prefix) {
-				return col.Name(), nil
-			}
-		}
-		return "", fmt.Errorf("no column found matching pattern '%s'", pattern)
-	}
-
-	for _, col := range columns {
-		if col.Name() == pattern {
-			return col.Name(), nil
-		}
-	}
-	return "", fmt.Errorf("column '%s' not found in table columns", pattern)
-}
-
-func (mw *mssqlDbWrapper) QueryFromObject(ctx context.Context, q ExtractionQuery) (RowsResult, error) {
-	queryString, err := buildExtractQueryMssql(q)
-	if err != nil {
-		return nil, err
-	}
-
-	logrus.Debugf("Query: %s", queryString)
-
-	var queryArgs []any
-
-	if q.LowerLimit.IsValid {
-		queryArgs = append(queryArgs, sql.Named("min", q.LowerLimit.Value))
-	}
-
-	if q.UpperLimit.IsValid {
-		queryArgs = append(queryArgs, sql.Named("max", q.UpperLimit.Value))
-	}
-
-	return mw.Query(ctx, queryString, queryArgs...)
-}
-
-func buildJsonPathMssql(field string) string {
-	if len(field) > 0 && field[0] == '.' {
-		field = field[1:]
-	}
-
-	return "$." + field
-}

internal/app/db-wrapper/mssql_test.go

@@ -1,396 +0,0 @@
-package dbwrapper
-
-import (
-	"strings"
-	"testing"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-func TestBuildExtractQueryMssql_NoJsonColumns(t *testing.T) {
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Users",
-		PrimaryKey: "ID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("Name", true, false, "VARCHAR", "varchar", "VARCHAR", true, 255, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{},
-		LowerLimit:      ExtractorQueryLimit{IsValid: false},
-		UpperLimit:      ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	if !strings.Contains(query, "SELECT [ID], [Name]") {
-		t.Errorf("Expected columns in query, got: %s", query)
-	}
-
-	if !strings.Contains(query, "FROM [dbo].[Users]") {
-		t.Errorf("Expected FROM clause, got: %s", query)
-	}
-
-	if !strings.Contains(query, "ORDER BY [ID] ASC") {
-		t.Errorf("Expected ORDER BY clause, got: %s", query)
-	}
-}
-
-func TestBuildExtractQueryMssql_WithJsonColumns_ExactColumnMatch(t *testing.T) {
-	// Test that the actual column name is used as alias, not a generated one
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Events",
-		PrimaryKey: "EventID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("EventID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("EventData", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "EventData", Field: ".userId"},
-			{Column: "EventData", Field: ".timestamp"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: false},
-		UpperLimit: ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	if !strings.HasPrefix(query, "SELECT [EventID], JSON_VALUE([EventData], '$.userId') AS [EventData], JSON_VALUE([EventData], '$.timestamp') AS [EventData]") {
-		t.Errorf("Expected JSON columns to replace EventData in-order, got: %s", query)
-	}
-
-	if strings.Contains(query, "SELECT [EventID], [EventData]") {
-		t.Errorf("Expected EventData to be replaced by JSON extraction, got: %s", query)
-	}
-
-	// Alias should be exactly "EventData", not "EventData_userId"
-	if !strings.Contains(query, "JSON_VALUE([EventData], '$.userId') AS [EventData]") {
-		t.Errorf("Expected JSON alias to be [EventData], got: %s", query)
-	}
-
-	if !strings.Contains(query, "JSON_VALUE([EventData], '$.timestamp') AS [EventData]") {
-		t.Errorf("Expected JSON alias to be [EventData], got: %s", query)
-	}
-
-	// Should have comma separating them
-	if !strings.Contains(query, "JSON_VALUE([EventData], '$.userId') AS [EventData], JSON_VALUE([EventData], '$.timestamp') AS [EventData]") {
-		t.Errorf("Expected comma-separated JSON values, got: %s", query)
-	}
-}
-
-func TestBuildExtractQueryMssql_WithWildcardPattern(t *testing.T) {
-	// Test that wildcard pattern matching finds the correct column
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Events",
-		PrimaryKey: "ID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("NodeMetadata", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "NodeMeta*", Field: ".id"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: false},
-		UpperLimit: ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	// Should find "NodeMetadata" from pattern "NodeMeta*" and use it as alias
-	if !strings.Contains(query, "JSON_VALUE([NodeMetadata], '$.id') AS [NodeMetadata]") {
-		t.Errorf("Expected to find and use NodeMetadata column by pattern, got: %s", query)
-	}
-
-	if strings.Contains(query, "SELECT [ID], [NodeMetadata]") {
-		t.Errorf("Expected NodeMetadata to be replaced by JSON extraction, got: %s", query)
-	}
-}
-
-func TestBuildExtractQueryMssql_ColumnNotFound_Error(t *testing.T) {
-	// Test that an error is returned when column is not found
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Events",
-		PrimaryKey: "ID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "NonExistentColumn", Field: ".id"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: false},
-		UpperLimit: ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err == nil {
-		t.Fatalf("Expected error for missing column, got no error. Query: %s", query)
-	}
-
-	if !strings.Contains(err.Error(), "NonExistentColumn") {
-		t.Errorf("Expected error message to contain column name, got: %v", err)
-	}
-}
-
-func TestBuildExtractQueryMssql_WildcardPatternNotMatched_Error(t *testing.T) {
-	// Test that an error is returned when wildcard pattern doesn't match any column
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Events",
-		PrimaryKey: "ID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("EventData", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "NonMatching*", Field: ".id"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: false},
-		UpperLimit: ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err == nil {
-		t.Fatalf("Expected error for non-matching wildcard pattern, got no error. Query: %s", query)
-	}
-
-	if !strings.Contains(err.Error(), "NonMatching*") {
-		t.Errorf("Expected error message to contain pattern, got: %v", err)
-	}
-}
-
-func TestBuildExtractQueryMssql_NestedJsonFields(t *testing.T) {
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Data",
-		PrimaryKey: "ID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("NodeData", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "NodeData", Field: ".user.name"},
-			{Column: "NodeData", Field: ".user.email"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: false},
-		UpperLimit: ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	if !strings.Contains(query, "JSON_VALUE([NodeData], '$.user.name') AS [NodeData]") {
-		t.Errorf("Expected nested JSON path for user.name, got: %s", query)
-	}
-
-	if !strings.Contains(query, "JSON_VALUE([NodeData], '$.user.email') AS [NodeData]") {
-		t.Errorf("Expected nested JSON path for user.email, got: %s", query)
-	}
-
-	if strings.Contains(query, "SELECT [ID], [NodeData]") {
-		t.Errorf("Expected NodeData to be replaced by JSON extraction, got: %s", query)
-	}
-}
-
-func TestBuildExtractQueryMssql_WithRangeLimits(t *testing.T) {
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Products",
-		PrimaryKey: "ProductID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("ProductID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("Details", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "Details", Field: ".price"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: true, IsInclusive: true, Value: 100},
-		UpperLimit: ExtractorQueryLimit{IsValid: true, IsInclusive: false, Value: 500},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	if !strings.Contains(query, "WHERE [ProductID] >= @min") {
-		t.Errorf("Expected WHERE clause with >=, got: %s", query)
-	}
-
-	if !strings.Contains(query, "[ProductID] < @max") {
-		t.Errorf("Expected upper limit with <, got: %s", query)
-	}
-
-	if !strings.Contains(query, "JSON_VALUE([Details], '$.price') AS [Details]") {
-		t.Errorf("Expected JSON_VALUE for Details, got: %s", query)
-	}
-
-	if strings.Contains(query, "SELECT [ProductID], [Details]") {
-		t.Errorf("Expected Details to be replaced by JSON extraction, got: %s", query)
-	}
-}
-
-func TestBuildJsonPathMssql(t *testing.T) {
-	tests := []struct {
-		input    string
-		expected string
-	}{
-		{".id", "$.id"},
-		{"id", "$.id"},
-		{".user.name", "$.user.name"},
-		{"user.name", "$.user.name"},
-		{".location.coordinates.lat", "$.location.coordinates.lat"},
-		{"", "$."},
-	}
-
-	for _, tt := range tests {
-		result := buildJsonPathMssql(tt.input)
-		if result != tt.expected {
-			t.Errorf("buildJsonPathMssql(%q) = %q, want %q", tt.input, result, tt.expected)
-		}
-	}
-}
-
-func TestFindColumnByPattern_ExactMatch(t *testing.T) {
-	columns := []models.ColumnType{
-		models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-		models.NewColumnType("Metadata", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		models.NewColumnType("EventData", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-	}
-
-	result, err := findColumnByPattern(columns, "Metadata")
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	if result != "Metadata" {
-		t.Errorf("Expected 'Metadata', got '%s'", result)
-	}
-}
-
-func TestFindColumnByPattern_WildcardMatch(t *testing.T) {
-	columns := []models.ColumnType{
-		models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-		models.NewColumnType("NodeMetadata", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		models.NewColumnType("EventData", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-	}
-
-	result, err := findColumnByPattern(columns, "NodeMeta*")
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	if result != "NodeMetadata" {
-		t.Errorf("Expected 'NodeMetadata', got '%s'", result)
-	}
-}
-
-func TestFindColumnByPattern_NotFound(t *testing.T) {
-	columns := []models.ColumnType{
-		models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-		models.NewColumnType("Metadata", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-	}
-
-	result, err := findColumnByPattern(columns, "NonExistent")
-	if err == nil {
-		t.Fatalf("Expected error, got no error. Result: %s", result)
-	}
-
-	if !strings.Contains(err.Error(), "NonExistent") {
-		t.Errorf("Expected error to contain column name, got: %v", err)
-	}
-}
-
-func TestFindColumnByPattern_WildcardNotFound(t *testing.T) {
-	columns := []models.ColumnType{
-		models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-		models.NewColumnType("Metadata", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-	}
-
-	result, err := findColumnByPattern(columns, "Event*")
-	if err == nil {
-		t.Fatalf("Expected error, got no error. Result: %s", result)
-	}
-
-	if !strings.Contains(err.Error(), "Event*") {
-		t.Errorf("Expected error to contain pattern, got: %v", err)
-	}
-}
-
-func TestBuildExtractQueryMssql_OnlyJsonColumns(t *testing.T) {
-	// Test when all columns are used via JSON extraction
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Data",
-		PrimaryKey: "ID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("ID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("JsonData", true, false, "VARCHAR", "varchar", "VARCHAR", true, 500, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "JsonData", Field: ".field1"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: false},
-		UpperLimit: ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	if !strings.HasPrefix(query, "SELECT [ID], JSON_VALUE([JsonData], '$.field1') AS [JsonData]") {
-		t.Errorf("Expected JsonData to be replaced by JSON extraction, got: %s", query)
-	}
-
-	if strings.Contains(query, "SELECT [ID], [JsonData]") {
-		t.Errorf("Expected JsonData to be excluded from raw selection, got: %s", query)
-	}
-}
-
-func TestBuildExtractQueryMssql_JsonColumnsReplaceInOrder(t *testing.T) {
-	q := ExtractionQuery{
-		Schema:     "dbo",
-		Table:      "Users",
-		PrimaryKey: "UserID",
-		Columns: []models.ColumnType{
-			models.NewColumnType("UserID", false, false, "INT", "int", "INT", false, 0, 0, 0),
-			models.NewColumnType("Name", true, false, "VARCHAR", "varchar", "VARCHAR", false, 255, 0, 0),
-			models.NewColumnType("Email", true, false, "VARCHAR", "varchar", "VARCHAR", false, 255, 0, 0),
-			models.NewColumnType("Metadata", true, false, "NVARCHAR", "nvarchar", "NVARCHAR", true, 4000, 0, 0),
-			models.NewColumnType("Profile", true, false, "NVARCHAR", "nvarchar", "NVARCHAR", true, 4000, 0, 0),
-			models.NewColumnType("Settings", true, false, "NVARCHAR", "nvarchar", "NVARCHAR", true, 4000, 0, 0),
-		},
-		FromJsonColumns: []config.FromJsonItem{
-			{Column: "Metadata", Field: ".id"},
-			{Column: "Profile", Field: ".id"},
-			{Column: "Settings", Field: ".id"},
-		},
-		LowerLimit: ExtractorQueryLimit{IsValid: false},
-		UpperLimit: ExtractorQueryLimit{IsValid: false},
-	}
-
-	query, err := buildExtractQueryMssql(q)
-	if err != nil {
-		t.Fatalf("Expected no error, got: %v", err)
-	}
-
-	expected := "SELECT [UserID], [Name], [Email], JSON_VALUE([Metadata], '$.id') AS [Metadata], JSON_VALUE([Profile], '$.id') AS [Profile], JSON_VALUE([Settings], '$.id') AS [Settings] FROM [dbo].[Users] ORDER BY [UserID] ASC"
-	if query != expected {
-		t.Errorf("Unexpected query.\nExpected: %s\nGot: %s", expected, query)
-	}
-}

internal/app/db-wrapper/postgres.go

@@ -3,17 +3,14 @@ package dbwrapper
 import (
 	"context"
 	"errors"
-	"fmt"
-	"strings"
 
-	dbdialects "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper/db_dialects"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 )
 
 func init() {
-	Register(dbdialects.Postgres, func() DbWrapper {
-		return &postgresDbWrapper{dialect: dbdialects.Postgres}
+	Register("postgres", func() DbWrapper {
+		return &postgresDbWrapper{dialect: "postgres"}
 	})
 }
 
@@ -129,75 +126,3 @@ func (pw *postgresDbWrapper) SaveMassive(ctx context.Context, schema string, tab
 
 	return affectedRows, nil
 }
-
-func (pw *postgresDbWrapper) QueryFromObject(ctx context.Context, q ExtractionQuery) (RowsResult, error) {
-	var sbQuery strings.Builder
-
-	sbQuery.WriteString("SELECT ")
-
-	if len(q.Columns) == 0 {
-		sbQuery.WriteString("*")
-	} else {
-		for i, col := range q.Columns {
-			switch col.Type() {
-			case "GEOMETRY":
-				fmt.Fprintf(&sbQuery, `ST_AsEWKB("%s") AS "%s"`, col.Name(), col.Name())
-			default:
-				fmt.Fprintf(&sbQuery, `"%s"`, col.Name())
-			}
-
-			if i < len(q.Columns)-1 {
-				sbQuery.WriteString(", ")
-			}
-		}
-	}
-
-	fmt.Fprintf(&sbQuery, ` FROM "%s"."%s"`, q.Schema, q.Table)
-
-	if q.LowerLimit.IsValid || q.UpperLimit.IsValid {
-		sbQuery.WriteString(" WHERE ")
-		paramIdx := 1
-
-		if q.LowerLimit.IsValid {
-			fmt.Fprintf(&sbQuery, `"%s"`, q.PrimaryKey)
-			if q.LowerLimit.IsInclusive {
-				sbQuery.WriteString(" >=")
-			} else {
-				sbQuery.WriteString(" >")
-			}
-			fmt.Fprintf(&sbQuery, " $%d", paramIdx)
-			paramIdx++
-		}
-
-		if q.LowerLimit.IsValid && q.UpperLimit.IsValid {
-			sbQuery.WriteString(" AND ")
-		}
-
-		if q.UpperLimit.IsValid {
-			fmt.Fprintf(&sbQuery, `"%s"`, q.PrimaryKey)
-			if q.UpperLimit.IsInclusive {
-				sbQuery.WriteString(" <=")
-			} else {
-				sbQuery.WriteString(" <")
-			}
-			fmt.Fprintf(&sbQuery, " $%d", paramIdx)
-			paramIdx++
-		}
-	}
-
-	fmt.Fprintf(&sbQuery, ` ORDER BY "%s" ASC`, q.PrimaryKey)
-
-	queryString := sbQuery.String()
-
-	var queryArgs []any
-
-	if q.LowerLimit.IsValid {
-		queryArgs = append(queryArgs, q.LowerLimit.Value)
-	}
-
-	if q.UpperLimit.IsValid {
-		queryArgs = append(queryArgs, q.UpperLimit.Value)
-	}
-
-	return pw.Query(ctx, queryString, queryArgs...)
-}

internal/app/db-wrapper/types.go

@@ -3,9 +3,6 @@ package dbwrapper
 import (
 	"context"
 	"errors"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 )
 
 var MethodNotSupported error = errors.New("Method not supported by driver... yet :P")
@@ -27,22 +24,6 @@ type RowResult interface {
 	Scan(dest ...any) error
 }
 
-type ExtractorQueryLimit struct {
-	IsValid     bool
-	IsInclusive bool
-	Value       int64
-}
-
-type ExtractionQuery struct {
-	Schema          string
-	Table           string
-	PrimaryKey      string
-	Columns         []models.ColumnType
-	LowerLimit      ExtractorQueryLimit
-	UpperLimit      ExtractorQueryLimit
-	FromJsonColumns []config.FromJsonItem
-}
-
 type DbWrapper interface {
 	Close() error
 	Connect(ctx context.Context, dbUrl string) error
@@ -51,5 +32,4 @@ type DbWrapper interface {
 	Query(ctx context.Context, query string, args ...any) (RowsResult, error)
 	QueryRow(ctx context.Context, query string, args ...any) RowResult
 	SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error)
-	QueryFromObject(ctx context.Context, query ExtractionQuery) (RowsResult, error)
 }

internal/app/etl/extractors/consume.go

@@ -1,96 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"errors"
-	"slices"
-	"strings"
-	"sync"
-	"sync/atomic"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/sirupsen/logrus"
-)
-
-func (ex *GenericExtractor) Consume(
-	ctx context.Context,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	retryConfig config.RetryConfig,
-	chPartitionsIn <-chan models.Partition,
-	chBatchesOut chan<- models.Batch,
-	chErrorsOut chan<- custom_errors.JobError,
-	wgActivePartitions *sync.WaitGroup,
-	rowsRead *int64,
-	fromJsonColumns []config.FromJsonItem,
-) {
-	indexPrimaryKey := slices.IndexFunc(columns, func(col models.ColumnType) bool {
-		return strings.EqualFold(col.Name(), tableInfo.PrimaryKey)
-	})
-
-	if indexPrimaryKey == -1 {
-		select {
-		case <-ctx.Done():
-			return
-		case chErrorsOut <- custom_errors.JobError{
-			ShouldCancelJob: true,
-			Msg:             "Primary key not found in provided columns",
-		}:
-		}
-
-		return
-	}
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		case partition, ok := <-chPartitionsIn:
-			if !ok {
-				return
-			}
-
-			rowsReadResult, err := ex.ProcessPartitionWithRetries(
-				ctx,
-				tableInfo,
-				columns,
-				batchSize,
-				partition,
-				indexPrimaryKey,
-				retryConfig,
-				chBatchesOut,
-				fromJsonColumns,
-			)
-			wgActivePartitions.Done()
-
-			if rowsReadResult > 0 {
-				current := atomic.LoadInt64(rowsRead)
-				logrus.Debugf("Rows read: +%v [current=%v] (%s.%s)", rowsReadResult, current, tableInfo.Schema, tableInfo.Table)
-				atomic.AddInt64(rowsRead, int64(rowsReadResult))
-			}
-
-			if err != nil {
-				if jobError, ok := errors.AsType[*custom_errors.JobError](err); ok {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- *jobError:
-					}
-				} else {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: false, Msg: err.Error(), Prev: err}:
-					}
-				}
-			}
-		}
-	}
-}

internal/app/etl/extractors/main.go

@@ -1,42 +0,0 @@
-package extractors
-
-import (
-	"context"
-
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-type GenericExtractor struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewExtractor(db dbwrapper.DbWrapper) GenericExtractor {
-	return GenericExtractor{db: db}
-}
-
-func sendBatch(ctx context.Context, chBatchesOut chan<- models.Batch, batch models.Batch) error {
-	select {
-	case chBatchesOut <- batch:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}
-
-func flush(
-	ctx context.Context,
-	partition *models.Partition,
-	batchSize int,
-	batchRows []models.UnknownRowValues,
-	chBatchesOut chan<- models.Batch,
-) error {
-	if len(batchRows) == 0 {
-		return nil
-	}
-
-	batch := models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows}
-	batchRows = make([]models.UnknownRowValues, 0, batchSize)
-	return sendBatch(ctx, chBatchesOut, batch)
-}

internal/app/etl/extractors/mssql.go

@@ -0,0 +1,277 @@
+package extractors
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/convert"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/google/uuid"
+)
+
+type MssqlExtractor struct {
+	db dbwrapper.DbWrapper
+}
+
+func NewMssqlExtractor(db dbwrapper.DbWrapper) etl.Extractor {
+	return &MssqlExtractor{db: db}
+}
+
+func buildExtractQueryMssql(
+	tableInfo config.SourceTableInfo,
+	columns []models.ColumnType,
+	includeRange bool,
+	isMinInclusive bool,
+) string {
+	var sbQuery strings.Builder
+
+	sbQuery.WriteString("SELECT ")
+
+	if len(columns) == 0 {
+		sbQuery.WriteString("*")
+	} else {
+		for i, col := range columns {
+			fmt.Fprintf(&sbQuery, "[%s]", col.Name())
+
+			if col.Type() == "GEOMETRY" {
+				fmt.Fprintf(&sbQuery, ".STAsBinary() AS [%s]", col.Name())
+			}
+
+			if i < len(columns)-1 {
+				sbQuery.WriteString(", ")
+			}
+		}
+	}
+
+	fmt.Fprintf(&sbQuery, " FROM [%s].[%s]", tableInfo.Schema, tableInfo.Table)
+
+	if includeRange {
+		fmt.Fprintf(&sbQuery, " WHERE [%s]", tableInfo.PrimaryKey)
+		if isMinInclusive {
+			sbQuery.WriteString(" >=")
+		} else {
+			sbQuery.WriteString(" >")
+		}
+
+		fmt.Fprintf(&sbQuery, " @min AND [%s] <= @max", tableInfo.PrimaryKey)
+	}
+
+	fmt.Fprintf(&sbQuery, " ORDER BY [%s] ASC", tableInfo.PrimaryKey)
+
+	return sbQuery.String()
+}
+
+func errorFromLastRow(
+	lastRow models.UnknownRowValues,
+	indexPrimaryKey int,
+	partition models.Partition,
+	previousError error,
+) *custom_errors.ExtractorError {
+	lastIdRawValue := lastRow[indexPrimaryKey]
+
+	lastId, ok := convert.ToInt64(lastIdRawValue)
+	if !ok {
+		currentPartition := partition
+		currentPartition.RetryCounter = 3
+		return &custom_errors.ExtractorError{
+			Partition: currentPartition,
+			HasLastId: true,
+			Msg:       fmt.Sprintf("Couldn't cast last id value as int: %s", previousError.Error()),
+		}
+
+	}
+
+	return &custom_errors.ExtractorError{
+		Partition: partition,
+		HasLastId: true,
+		LastId:    lastId,
+		Msg:       previousError.Error(),
+	}
+}
+
+func (mssqlEx *MssqlExtractor) ProcessPartition(
+	ctx context.Context,
+	tableInfo config.SourceTableInfo,
+	columns []models.ColumnType,
+	batchSize int,
+	partition models.Partition,
+	indexPrimaryKey int,
+	chBatchesOut chan<- models.Batch,
+) (int, error) {
+	query := buildExtractQueryMssql(tableInfo, columns, partition.HasRange, partition.Range.IsMinInclusive)
+
+	var queryArgs []any
+	if partition.HasRange {
+		queryArgs = append(queryArgs,
+			sql.Named("min", partition.Range.Min),
+			sql.Named("max", partition.Range.Max),
+		)
+	}
+
+	rowsRead := 0
+	rows, err := mssqlEx.db.Query(ctx, query, queryArgs...)
+	if err != nil {
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+	}
+	defer rows.Close()
+
+	batchRows := make([]models.UnknownRowValues, 0, batchSize)
+
+	for rows.Next() {
+		rowValues := make([]any, len(columns))
+		scanArgs := make([]any, len(columns))
+
+		for i := range rowValues {
+			scanArgs[i] = &rowValues[i]
+		}
+
+		if err := rows.Scan(scanArgs...); err != nil {
+			if len(batchRows) == 0 {
+				return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+			}
+
+			lastRow := batchRows[len(batchRows)-1]
+
+			select {
+			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+			case <-ctx.Done():
+				return rowsRead, ctx.Err()
+			}
+
+			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
+		}
+		rowsRead++
+
+		batchRows = append(batchRows, rowValues)
+		if len(batchRows) >= batchSize {
+			select {
+			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+			case <-ctx.Done():
+				return rowsRead, ctx.Err()
+			}
+
+			batchRows = make([]models.UnknownRowValues, 0, batchSize)
+		}
+
+	}
+
+	if err := rows.Err(); err != nil {
+		if errors.Is(err, ctx.Err()) {
+			return rowsRead, ctx.Err()
+		}
+
+		if len(batchRows) > 0 {
+			lastRow := batchRows[len(batchRows)-1]
+			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
+		}
+
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+	}
+
+	if len(batchRows) > 0 {
+		select {
+		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+		case <-ctx.Done():
+			return rowsRead, ctx.Err()
+		}
+	}
+
+	return rowsRead, nil
+}
+
+func (mssqlEx *MssqlExtractor) Exec(
+	ctx context.Context,
+	tableInfo config.SourceTableInfo,
+	columns []models.ColumnType,
+	batchSize int,
+	chPartitionsIn <-chan models.Partition,
+	chBatchesOut chan<- models.Batch,
+	chErrorsOut chan<- custom_errors.ExtractorError,
+	chJobErrorsOut chan<- custom_errors.JobError,
+	wgActivePartitions *sync.WaitGroup,
+	rowsRead *int64,
+) {
+	indexPrimaryKey := slices.IndexFunc(columns, func(col models.ColumnType) bool {
+		return strings.EqualFold(col.Name(), tableInfo.PrimaryKey)
+	})
+
+	if indexPrimaryKey == -1 {
+		select {
+		case <-ctx.Done():
+			return
+		case chJobErrorsOut <- custom_errors.JobError{
+			ShouldCancelJob: true,
+			Msg:             "Primary key not found in provided columns",
+		}:
+		}
+
+		return
+	}
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case partition, ok := <-chPartitionsIn:
+			if !ok {
+				return
+			}
+
+			rowsReadResult, err := mssqlEx.ProcessPartition(
+				ctx,
+				tableInfo,
+				columns,
+				batchSize,
+				partition,
+				indexPrimaryKey,
+				chBatchesOut,
+			)
+
+			if rowsReadResult > 0 {
+				atomic.AddInt64(rowsRead, int64(rowsReadResult))
+			}
+
+			if err != nil {
+				var exError *custom_errors.ExtractorError
+				var jobError *custom_errors.JobError
+				if errors.As(err, &exError) {
+					select {
+					case <-ctx.Done():
+						return
+					case chErrorsOut <- *exError:
+					}
+				} else if errors.As(err, &jobError) {
+					select {
+					case <-ctx.Done():
+						return
+					case chJobErrorsOut <- *jobError:
+					}
+				} else {
+					select {
+					case <-ctx.Done():
+						return
+					case chErrorsOut <- custom_errors.ExtractorError{Partition: partition, Msg: err.Error()}:
+					}
+				}
+
+				continue
+			}
+
+			wgActivePartitions.Done()
+		}
+	}
+}

internal/app/etl/extractors/postgres.go

@@ -0,0 +1,125 @@
+package extractors
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/google/uuid"
+)
+
+type PostgresExtractor struct {
+	db dbwrapper.DbWrapper
+}
+
+func NewPostgresExtractor(db dbwrapper.DbWrapper) etl.Extractor {
+	return &PostgresExtractor{db: db}
+}
+
+func buildExtractQueryPostgres(sourceDbInfo config.SourceTableInfo, columns []models.ColumnType) string {
+	var sbColumns strings.Builder
+
+	if len(columns) == 0 {
+		sbColumns.WriteString("*")
+	} else {
+		for i, col := range columns {
+			if col.Type() == "GEOMETRY" {
+				sbColumns.WriteString(`ST_AsEWKB("`)
+				sbColumns.WriteString(col.Name())
+				sbColumns.WriteString(`") AS "`)
+				sbColumns.WriteString(col.Name())
+				sbColumns.WriteString(`"`)
+			} else {
+				sbColumns.WriteString(`"`)
+				sbColumns.WriteString(col.Name())
+				sbColumns.WriteString(`"`)
+			}
+
+			if i < len(columns)-1 {
+				sbColumns.WriteString(", ")
+			}
+		}
+	}
+
+	return fmt.Sprintf(`SELECT %s FROM "%s"."%s"  ORDER BY "%s" ASC`, sbColumns.String(), sourceDbInfo.Schema, sourceDbInfo.Table, sourceDbInfo.PrimaryKey)
+}
+
+func (postgresEx *PostgresExtractor) ProcessPartition(
+	ctx context.Context,
+	tableInfo config.SourceTableInfo,
+	columns []models.ColumnType,
+	batchSize int,
+	partition models.Partition,
+	indexPrimaryKey int,
+	chBatchesOut chan<- models.Batch,
+) (int, error) {
+	query := buildExtractQueryPostgres(tableInfo, columns)
+
+	if partition.HasRange {
+		return 0, errors.New("Batch config not yet supported")
+	}
+
+	rowsRead := 0
+	rows, err := postgresEx.db.Query(ctx, query)
+	if err != nil {
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+	}
+	defer rows.Close()
+
+	batchRows := make([]models.UnknownRowValues, 0, batchSize)
+
+	for rows.Next() {
+		values, err := rows.Values()
+		if err != nil {
+			return rowsRead, errors.New("Unexpected error reading rows from source")
+		}
+		rowsRead++
+
+		batchRows = append(batchRows, values)
+
+		if len(batchRows) >= batchSize {
+			select {
+			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+			case <-ctx.Done():
+				return rowsRead, ctx.Err()
+			}
+
+			batchRows = make([]models.UnknownRowValues, 0, batchSize)
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return rowsRead, errors.New("Unexpected error reading rows from source")
+	}
+
+	if len(batchRows) > 0 {
+		select {
+		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+		case <-ctx.Done():
+			return rowsRead, nil
+		}
+	}
+
+	return rowsRead, nil
+}
+
+func (postgresEx *PostgresExtractor) Exec(
+	ctx context.Context,
+	tableInfo config.SourceTableInfo,
+	columns []models.ColumnType,
+	batchSize int,
+	chPartitionsIn <-chan models.Partition,
+	chBatchesOut chan<- models.Batch,
+	chErrorsOut chan<- custom_errors.ExtractorError,
+	chJobErrorsOut chan<- custom_errors.JobError,
+	wgActivePartitions *sync.WaitGroup,
+	rowsRead *int64,
+) {
+}

internal/app/etl/extractors/process-with-retries.go

@@ -1,77 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-	// "github.com/sirupsen/logrus"
-)
-
-func (ex *GenericExtractor) ProcessPartitionWithRetries(
-	ctx context.Context,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	partition models.Partition,
-	indexPrimaryKey int,
-	retryConfig config.RetryConfig,
-	chBatchesOut chan<- models.Batch,
-	fromJsonColumns []config.FromJsonItem,
-) (int64, error) {
-	var totalRowsRead int64
-	currentParitition := partition
-
-	for {
-		rowsRead, err := ex.ProcessPartition(
-			ctx,
-			tableInfo,
-			columns,
-			batchSize,
-			currentParitition,
-			indexPrimaryKey,
-			chBatchesOut,
-			fromJsonColumns,
-		)
-		// logrus.Debugf("Partition %v finished processing (%s.%s)", partition.Id, tableInfo.Schema, tableInfo.Table)
-		totalRowsRead += rowsRead
-
-		if err == nil {
-			return totalRowsRead, nil
-		}
-
-		if exError, ok := errors.AsType[*custom_errors.ExtractorError](err); ok {
-			currentParitition.RetryCounter++
-
-			if currentParitition.RetryCounter >= retryConfig.Attempts {
-				return totalRowsRead, &custom_errors.JobError{
-					Msg:  fmt.Sprintf("Partition %v reached max retries", exError.Partition.Id),
-					Prev: err,
-				}
-			}
-
-			if exError.HasLastId {
-				currentParitition.ParentId = exError.Partition.Id
-				currentParitition.Id = uuid.New()
-				currentParitition.Range.Min = exError.LastId
-				currentParitition.Range.IsMinInclusive = false
-			}
-
-			delay := custom_errors.ComputeBackoffDelay(
-				currentParitition.RetryCounter,
-				retryConfig.BaseDelayMs,
-				retryConfig.MaxDelayMs,
-				retryConfig.MaxJitterMs,
-			)
-			time.Sleep(delay)
-			continue
-		}
-
-		return totalRowsRead, err
-	}
-}

internal/app/etl/extractors/process.go

@@ -1,118 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"fmt"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/convert"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	// "github.com/sirupsen/logrus"
-)
-
-func errorFromLastPartitionRow(
-	lastRow models.UnknownRowValues,
-	indexPrimaryKey int,
-	partition models.Partition,
-	previousError error,
-) error {
-	lastIdRawValue := lastRow[indexPrimaryKey]
-
-	lastId, ok := convert.ToInt64(lastIdRawValue)
-	if !ok {
-		currentPartition := partition
-		currentPartition.RetryCounter = 3
-		return &custom_errors.ExtractorError{
-			Partition: currentPartition,
-			HasLastId: true,
-			Msg:       fmt.Sprintf("Couldn't cast last id value as int: %s", previousError.Error()),
-		}
-	}
-
-	return &custom_errors.ExtractorError{
-		Partition: partition,
-		HasLastId: true,
-		LastId:    lastId,
-		Msg:       previousError.Error(),
-	}
-}
-
-func (ex *GenericExtractor) ProcessPartition(
-	ctx context.Context,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	partition models.Partition,
-	indexPrimaryKey int,
-	chBatchesOut chan<- models.Batch,
-	fromJsonColumns []config.FromJsonItem,
-) (int64, error) {
-	query := dbwrapper.ExtractionQuery{
-		Schema:     tableInfo.Schema,
-		Table:      tableInfo.Table,
-		PrimaryKey: tableInfo.PrimaryKey,
-		Columns:    columns,
-		LowerLimit: dbwrapper.ExtractorQueryLimit{
-			IsValid:     partition.HasRange && partition.Range.Min > 0,
-			IsInclusive: partition.Range.IsMinInclusive,
-			Value:       partition.Range.Min,
-		},
-		UpperLimit: dbwrapper.ExtractorQueryLimit{
-			IsValid:     partition.HasRange && partition.Range.Max > 0,
-			IsInclusive: partition.Range.IsMaxInclusive,
-			Value:       partition.Range.Max,
-		},
-		FromJsonColumns: fromJsonColumns,
-	}
-
-	// logrus.Debugf("Processing partition: %+v (%s.%s)", query, tableInfo.Schema, tableInfo.Table)
-	rows, err := ex.db.QueryFromObject(ctx, query)
-	if err != nil {
-		return 0, err
-	}
-	defer rows.Close()
-
-	batchRows := make([]models.UnknownRowValues, 0, batchSize)
-	var rowsRead int64 = 0
-
-	for rows.Next() {
-		rowValues := make([]any, len(columns))
-		scanArgs := make([]any, len(columns))
-
-		for i := range rowValues {
-			scanArgs[i] = &rowValues[i]
-		}
-
-		if err := rows.Scan(scanArgs...); err != nil {
-			if len(batchRows) == 0 {
-				return rowsRead, err
-			}
-
-			if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
-				return rowsRead, err
-			}
-
-			lastRow := batchRows[len(batchRows)-1]
-			return rowsRead, errorFromLastPartitionRow(lastRow, indexPrimaryKey, partition, err)
-		}
-		rowsRead++
-
-		batchRows = append(batchRows, rowValues)
-		if len(batchRows) >= batchSize {
-			// logrus.Debugf("Batch size reached, flushing batch with %v rows (rowsRead=%v)", len(batchRows), rowsRead)
-			if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
-				// logrus.Warnf("Error flushing rows: %v", err)
-				return rowsRead, err
-			}
-			batchRows = make([]models.UnknownRowValues, 0, batchSize)
-		}
-	}
-
-	if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
-		return rowsRead, err
-	}
-
-	return rowsRead, rows.Err()
-}

internal/app/etl/loaders/consume.go

@@ -1,63 +0,0 @@
-package loaders
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"sync/atomic"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-func (gl *GenericLoader) Consume(
-	ctx context.Context,
-	tableInfo config.TargetTableInfo,
-	columns []models.ColumnType,
-	retryConfig config.RetryConfig,
-	chBatchesIn <-chan models.Batch,
-	chErrorsOut chan<- custom_errors.JobError,
-	wgActiveBatches *sync.WaitGroup,
-	rowsLoaded *int64,
-) {
-	colNames := mapSlice(columns, func(col models.ColumnType) string {
-		return col.Name()
-	})
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		case batch, ok := <-chBatchesIn:
-			if !ok {
-				return
-			}
-
-			processedRows, err := gl.ProcessBatchWithRetries(ctx, tableInfo, colNames, retryConfig, batch)
-			wgActiveBatches.Done()
-
-			if err != nil {
-				if jobError, ok := errors.AsType[*custom_errors.JobError](err); ok {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- *jobError:
-					}
-				} else {
-					select {
-					case <-ctx.Done():
-						return
-					case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: false, Msg: err.Error(), Prev: err}:
-					}
-				}
-			} else {
-				atomic.AddInt64(rowsLoaded, int64(processedRows))
-			}
-		}
-	}
-}

internal/app/etl/loaders/main.go

@@ -1,13 +0,0 @@
-package loaders
-
-import (
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-)
-
-type GenericLoader struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewGenericLoader(db dbwrapper.DbWrapper) GenericLoader {
-	return GenericLoader{db: db}
-}

internal/app/etl/loaders/postgres.go

@@ -0,0 +1,127 @@
+package loaders
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/jackc/pgx/v5/pgconn"
+)
+
+type PostgresLoader struct {
+	db dbwrapper.DbWrapper
+}
+
+func NewPostgresLoader(db dbwrapper.DbWrapper) etl.Loader {
+	return &PostgresLoader{db: db}
+}
+
+func mapSlice[T any, V any](input []T, mapper func(T) V) []V {
+	result := make([]V, len(input))
+
+	for i, v := range input {
+		result[i] = mapper(v)
+	}
+
+	return result
+}
+
+func (postgresLd *PostgresLoader) ProcessBatch(
+	ctx context.Context,
+	tableInfo config.TargetTableInfo,
+	colNames []string,
+	batch models.Batch,
+) (int, error) {
+	_, err := postgresLd.db.SaveMassive(
+		ctx,
+		tableInfo.Schema,
+		tableInfo.Table,
+		colNames,
+		batch.Rows,
+	)
+
+	if err != nil {
+		var pgErr *pgconn.PgError
+		if errors.As(err, &pgErr) {
+			if pgErr.Code == "23505" {
+				return 0, &custom_errors.JobError{
+					ShouldCancelJob: true,
+					Msg:             fmt.Sprintf("Fatal error in table %s.%s", tableInfo.Schema, tableInfo.Table),
+					Prev:            err,
+				}
+			}
+		}
+
+		return 0, &custom_errors.LoaderError{Batch: batch, Msg: err.Error()}
+	}
+
+	return len(batch.Rows), nil
+}
+
+func (postgresLd *PostgresLoader) Exec(
+	ctx context.Context,
+	tableInfo config.TargetTableInfo,
+	columns []models.ColumnType,
+	chBatchesIn <-chan models.Batch,
+	chErrorsOut chan<- custom_errors.LoaderError,
+	chJobErrorsOut chan<- custom_errors.JobError,
+	wgActiveBatches *sync.WaitGroup,
+	rowsLoaded *int64,
+) {
+	colNames := mapSlice(columns, func(col models.ColumnType) string {
+		return col.Name()
+	})
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case batch, ok := <-chBatchesIn:
+			if !ok {
+				return
+			}
+
+			processedRows, err := postgresLd.ProcessBatch(ctx, tableInfo, colNames, batch)
+
+			if err != nil {
+				var ldError *custom_errors.LoaderError
+				var jobError *custom_errors.JobError
+				if errors.As(err, &ldError) {
+					select {
+					case <-ctx.Done():
+						return
+					case chErrorsOut <- *ldError:
+					}
+				} else if errors.As(err, &jobError) {
+					select {
+					case <-ctx.Done():
+						return
+					case chJobErrorsOut <- *jobError:
+					}
+				} else {
+					select {
+					case <-ctx.Done():
+						return
+					case chErrorsOut <- custom_errors.LoaderError{Batch: batch, Msg: err.Error()}:
+					}
+				}
+
+				continue
+			}
+
+			wgActiveBatches.Done()
+			atomic.AddInt64(rowsLoaded, int64(processedRows))
+		}
+	}
+}

internal/app/etl/loaders/process-with-retries.go

@@ -1,50 +0,0 @@
-package loaders
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"time"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-)
-
-func (gl *GenericLoader) ProcessBatchWithRetries(
-	ctx context.Context,
-	tableInfo config.TargetTableInfo,
-	colNames []string,
-	retryConfig config.RetryConfig,
-	batch models.Batch,
-) (int64, error) {
-	for {
-		rowsLoaded, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch)
-		if err == nil {
-			return rowsLoaded, nil
-		}
-
-		if btError, ok := errors.AsType[*custom_errors.LoaderError](err); ok {
-			batch.RetryCounter++
-
-			if batch.RetryCounter >= retryConfig.Attempts {
-				return rowsLoaded, &custom_errors.JobError{
-					ShouldCancelJob: false,
-					Msg:             fmt.Sprintf("Temporal error in batch %v (retries: %d)", btError.Batch.Id, btError.Batch.RetryCounter),
-					Prev:            btError,
-				}
-			}
-
-			delay := custom_errors.ComputeBackoffDelay(
-				batch.RetryCounter,
-				retryConfig.BaseDelayMs,
-				retryConfig.MaxDelayMs,
-				retryConfig.MaxJitterMs,
-			)
-			time.Sleep(delay)
-			continue
-		}
-
-		return rowsLoaded, err
-	}
-}

internal/app/etl/loaders/process.go

@@ -1,43 +0,0 @@
-package loaders
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/jackc/pgx/v5/pgconn"
-)
-
-func (gl *GenericLoader) ProcessBatch(
-	ctx context.Context,
-	tableInfo config.TargetTableInfo,
-	colNames []string,
-	batch models.Batch,
-) (int64, error) {
-	_, err := gl.db.SaveMassive(
-		ctx,
-		tableInfo.Schema,
-		tableInfo.Table,
-		colNames,
-		batch.Rows,
-	)
-
-	if err != nil {
-		if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
-			if pgErr.Code == "23505" {
-				return 0, &custom_errors.JobError{
-					ShouldCancelJob: true,
-					Msg:             fmt.Sprintf("Fatal error in table %s.%s", tableInfo.Schema, tableInfo.Table),
-					Prev:            err,
-				}
-			}
-		}
-
-		return 0, &custom_errors.LoaderError{Batch: batch, Msg: err.Error()}
-	}
-
-	return int64(len(batch.Rows)), nil
-}

internal/app/etl/loaders/types.go

@@ -0,0 +1 @@
+package loaders

internal/app/etl/loaders/utils.go

@@ -1,11 +0,0 @@
-package loaders
-
-func mapSlice[T any, V any](input []T, mapper func(T) V) []V {
-	result := make([]V, len(input))
-
-	for i, v := range input {
-		result[i] = mapper(v)
-	}
-
-	return result
-}

internal/app/etl/table_analyzers/main.go

@@ -7,7 +7,6 @@ import (
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	"github.com/google/uuid"
-	"github.com/sirupsen/logrus"
 )
 
 func PartitionRangeGenerator(
@@ -16,24 +15,8 @@ func PartitionRangeGenerator(
 	tableInfo config.TableInfo,
 	partitionColumn string,
 	rowsPerPartition int64,
-	jobRange config.RangeConfig,
 ) ([]models.Partition, error) {
-	if jobRange.Min > 0 {
-		return []models.Partition{{
-			Id:           uuid.New(),
-			HasRange:     true,
-			RetryCounter: 0,
-			Range: models.PartitionRange{
-				Min:            jobRange.Min,
-				Max:            jobRange.Max,
-				IsMinInclusive: jobRange.IsMinInclusive,
-				IsMaxInclusive: jobRange.IsMaxInclusive,
-			},
-		}}, nil
-	}
-
 	rowsCount, err := tableAnalyzer.EstimateTotalRows(ctx, tableInfo)
-	logrus.Infof("Estimated rows in source: %v (%s.%s)", rowsCount, tableInfo.Schema, tableInfo.Table)
 	if err != nil {
 		return nil, err
 	}
@@ -53,7 +36,5 @@ func PartitionRangeGenerator(
 		return nil, err
 	}
 
-	// logrus.Debugf("Partitions: %+v (%s.%s)", partitions, tableInfo.Schema, tableInfo.Table)
-
 	return partitions, nil
 }

internal/app/etl/table_analyzers/mssql.go

@@ -234,7 +234,6 @@ ORDER BY batch_id`,
 			RetryCounter: 0,
 			Range: models.PartitionRange{
 				IsMinInclusive: true,
-				IsMaxInclusive: true,
 			},
 		}
 

internal/app/etl/transformers/mssql.go

@@ -3,32 +3,18 @@ package transformers
 import (
 	"context"
 	"errors"
-	"fmt"
-	"strings"
 	"sync"
 	"time"
 
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/azure"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-	log "github.com/sirupsen/logrus"
 )
 
-type MssqlTransformer struct {
-	toStorage   config.ToStorageConfig
-	sourceTable config.SourceTableInfo
-	azureClient *azure.Client
-}
+type MssqlTransformer struct{}
 
-func NewMssqlTransformer(toStorage config.ToStorageConfig, sourceTable config.SourceTableInfo, azureClient *azure.Client) etl.Transformer {
-	return &MssqlTransformer{
-		toStorage:   toStorage,
-		sourceTable: sourceTable,
-		azureClient: azureClient,
-	}
+func NewMssqlTransformer() etl.Transformer {
+	return &MssqlTransformer{}
 }
 
 func computeTransformationPlan(columns []models.ColumnType) []etl.ColumnTransformPlan {
@@ -74,65 +60,6 @@ func computeTransformationPlan(columns []models.ColumnType) []etl.ColumnTransfor
 	return plan
 }
 
-func computeStorageTransformationPlan(
-	ctx context.Context,
-	azureClient *azure.Client,
-	toStorage config.ToStorageConfig,
-	sourceColumns []models.ColumnType,
-	sourceTable config.SourceTableInfo,
-) []etl.ColumnTransformPlan {
-	if azureClient == nil || len(toStorage.Columns) == 0 {
-		return nil
-	}
-
-	colIndex := make(map[string]int, len(sourceColumns))
-	for i, col := range sourceColumns {
-		colIndex[strings.ToUpper(col.Name())] = i
-	}
-
-	var plan []etl.ColumnTransformPlan
-	for _, storageCol := range toStorage.Columns {
-		if storageCol.Mode != "REFERENCE_ONLY" {
-			log.Warnf("to_storage: unsupported mode %q for column %s — skipping", storageCol.Mode, storageCol.Source)
-			continue
-		}
-
-		idx, ok := colIndex[strings.ToUpper(storageCol.Source)]
-		if !ok {
-			log.Warnf("to_storage: source column %q not found in source schema — skipping", storageCol.Source)
-			continue
-		}
-
-		sourceColName := storageCol.Source
-		schema := sourceTable.Schema
-		table := sourceTable.Table
-
-		plan = append(plan, etl.ColumnTransformPlan{
-			Index: idx,
-			Fn: func(v any) (any, error) {
-				if v == nil {
-					return nil, nil
-				}
-				b, ok := v.([]byte)
-				if !ok {
-					log.Warnf("to_storage: expected []byte for %s.%s.%s, got %T — passing through",
-						schema, table, sourceColName, v)
-					return v, nil
-				}
-				start := time.Now()
-				blobPath := fmt.Sprintf("%s/%s/%s", schema, table, uuid.New().String())
-				blobURL, err := azureClient.UploadAndGetURL(ctx, blobPath, b)
-				if err != nil {
-					return nil, fmt.Errorf("uploading %s.%s.%s: %w", schema, table, sourceColName, err)
-				}
-				log.Debugf(`Succesfully uploaded "%s", (%vms)`, blobURL, time.Since(start).Milliseconds())
-				return blobURL, nil
-			},
-		})
-	}
-	return plan
-}
-
 const processBatchCtxCheck = 4096
 
 func (mssqlTr *MssqlTransformer) ProcessBatch(
@@ -173,8 +100,6 @@ func (mssqlTr *MssqlTransformer) Exec(
 	wgActiveBatches *sync.WaitGroup,
 ) {
 	transformationPlan := computeTransformationPlan(columns)
-	storagePlan := computeStorageTransformationPlan(ctx, mssqlTr.azureClient, mssqlTr.toStorage, columns, mssqlTr.sourceTable)
-	transformationPlan = append(transformationPlan, storagePlan...)
 
 	for {
 		if ctx.Err() != nil {

internal/app/etl/types.go

@@ -19,6 +19,19 @@ type Extractor interface {
 		indexPrimaryKey int,
 		chBatchesOut chan<- models.Batch,
 	) (int, error)
+
+	Exec(
+		ctx context.Context,
+		tableInfo config.SourceTableInfo,
+		columns []models.ColumnType,
+		batchSize int,
+		chPartitionsIn <-chan models.Partition,
+		chBatchesOut chan<- models.Batch,
+		chErrorsOut chan<- custom_errors.ExtractorError,
+		chJobErrorsOut chan<- custom_errors.JobError,
+		wgActivePartitions *sync.WaitGroup,
+		rowsRead *int64,
+	)
 }
 
 type TransformerFunc func(any) (any, error)
@@ -51,7 +64,18 @@ type Loader interface {
 		tableInfo config.TargetTableInfo,
 		colNames []string,
 		batch models.Batch,
-	) (int64, error)
+	) (int, error)
+
+	Exec(
+		ctx context.Context,
+		tableInfo config.TargetTableInfo,
+		columns []models.ColumnType,
+		chBatchesIn <-chan models.Batch,
+		chErrorsOut chan<- custom_errors.LoaderError,
+		chJobErrorsOut chan<- custom_errors.JobError,
+		wgActiveBatches *sync.WaitGroup,
+		rowsLoaded *int64,
+	)
 }
 
 type TableAnalyzer interface {

scripts/az-blob/main.go

@@ -1,44 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"math/rand"
-	"sync"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/azure"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-)
-
-func main() {
-	cfg := config.App.AzureStorage
-	containerName := cfg.Container
-
-	client, err := azure.NewClient(cfg)
-	if err != nil {
-		log.Fatalf("Error creando cliente: %v", err)
-	}
-	ctx := context.Background()
-
-	var wg sync.WaitGroup
-
-	for i := 1; i <= 10; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-
-			blobName := fmt.Sprintf("%sarchivo-%d.txt", cfg.Prefix, id)
-			content := fmt.Sprintf("Contenido aleatorio: %d", rand.Intn(100000))
-
-			err := client.UploadBuffer(ctx, containerName, blobName, []byte(content))
-			if err != nil {
-				log.Printf("Fallo al subir %s: %v", blobName, err)
-			} else {
-				fmt.Printf("Subido exitosamente: %s\n", blobName)
-			}
-		}(i)
-	}
-
-	wg.Wait()
-}

scripts/backoff-test/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/cenkalti/backoff/v5"
+)
+
+func ExampleRetry() {
+	// Define an operation function that returns a value and an error.
+	// The value can be any type.
+	// We'll pass this operation to Retry function.
+	operation := func() (string, error) {
+		// An example request that may fail.
+		resp, err := http.Get("http://httpbin.org/get")
+		if err != nil {
+			return "", err
+		}
+		defer resp.Body.Close()
+
+		// If we are being rate limited, return a RetryAfter to specify how long to wait.
+		// This will also reset the backoff policy.
+		if resp.StatusCode == 429 {
+			seconds, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64)
+			if err == nil {
+				return "", backoff.RetryAfter(int(seconds))
+			}
+		}
+
+		// In case of non-retriable error, return Permanent error to stop retrying.
+		// For this HTTP example, client errors are non-retriable.
+		if resp.StatusCode >= 400 && resp.StatusCode < 500 {
+			return "", backoff.Permanent(errors.New("bad request"))
+		}
+
+		// Return successful response.
+		return "hello", nil
+	}
+
+	result, err := backoff.Retry(context.TODO(), operation, backoff.WithBackOff(backoff.NewExponentialBackOff()))
+	if err != nil {
+		fmt.Println("Error:", err)
+		return
+	}
+
+	// Operation is successful after retries.
+
+	fmt.Println(result)
+	// Output: hello
+}
+
+func main() {
+	ExampleRetry()
+}