refactor: replace per-dialect extractors with generic extractor

Collapse the mssql- and postgres-specific extractors into a single generic implementation driven by the DB dialect interface.

cmd/go_migrate/main.go

@@ -118,7 +118,7 @@ func processMigrationJobs(
 
 	sourceTableAnalyzer := table_analyzers.NewMssqlTableAnalyzer(sourceDb)
 	targetTableAnalyzer := table_analyzers.NewPostgresTableAnalyzer(targetDb)
-	extractor := extractors.NewMssqlExtractor(sourceDb)
+	extractor := extractors.NewExtractor(sourceDb)
 	loader := loaders.NewGenericLoader(targetDb)
 
 	var azureClient *azure.Client

cmd/go_migrate/process.go

@@ -39,7 +39,7 @@ func processMigrationJob(
 	targetDbWrapper dbwrapper.DbWrapper,
 	sourceTableAnalyzer etl.TableAnalyzer,
 	targetTableAnalyzer etl.TableAnalyzer,
-	extractor etl.Extractor,
+	extractor extractors.GenericExtractor,
 	azureClient *azure.Client,
 	loader etl.Loader,
 	job config.Job,
@@ -155,9 +155,8 @@ func processMigrationJob(
 
 	for range maxExtractors {
 		wgExtractors.Go(func() {
-			extractors.Consume(
+			extractor.Consume(
 				localCtx,
-				extractor,
 				job.SourceTable,
 				sourceColTypes,
 				job.BatchSize,

config.yaml

@@ -50,16 +50,25 @@ jobs:
       post_sql: 
         - "SELECT 1"
 
-  - name: storage_attachments
+  - name: analytics_audit_log
     source:
-      schema: analytics
+      schema: storage
       table: attachments
       primary_key: id
     target:
-      schema: analytics
+      schema: storage
       table: attachments
     to_storage:
       columns:
         - source: DATA
           target: FILE_URL
-          mode: REFERENCE_ONLY # REFERENCE_ONLY | DUPLICATE_WITH_REF
+          mode: REFERENCE_ONLY
+    max_extractors: 8
+    max_loaders: 4
+    queue_size: 32
+    batch_size: 1
+    retry:
+      attempts: 5
+      base_delay_ms: 1000
+      max_delay_ms: 15000
+      max_jitter_ms: 500

internal/app/db-wrapper/db_dialects/main.go

@@ -0,0 +1,7 @@
+package db_dialects
+
+const (
+	SqlServer string = "sqlserver"
+	Postgres  string = "postgres"
+	Null      string = "null"
+)

internal/app/db-wrapper/mssql.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 
+	dbdialects "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper/db_dialects"
 	mssql "github.com/microsoft/go-mssqldb"
 )
 
 func init() {
-	Register("sqlserver", func() DbWrapper {
+	Register(dbdialects.SqlServer, func() DbWrapper {
-		return &mssqlDbWrapper{dialect: "sqlserver"}
+		return &mssqlDbWrapper{dialect: dbdialects.SqlServer}
 	})
 }
 
@@ -174,3 +176,72 @@ func (mw *mssqlDbWrapper) SaveMassive(ctx context.Context, schema string, table
 
 	return rowsAffected, nil
 }
+
+func (mw *mssqlDbWrapper) QueryFromObject(ctx context.Context, q ExtractionQuery) (RowsResult, error) {
+	var sbQuery strings.Builder
+
+	sbQuery.WriteString("SELECT ")
+
+	if len(q.columns) == 0 {
+		sbQuery.WriteString("*")
+	} else {
+		for i, col := range q.columns {
+			fmt.Fprintf(&sbQuery, "[%s]", col.Name())
+
+			switch col.Type() {
+			case "GEOMETRY":
+				fmt.Fprintf(&sbQuery, ".STAsBinary() AS [%s]", col.Name())
+			}
+
+			if i < len(q.columns)-1 {
+				sbQuery.WriteString(", ")
+			}
+		}
+	}
+
+	fmt.Fprintf(&sbQuery, " FROM [%s].[%s]", q.Schema, q.Table)
+
+	if q.LowerLimit.IsValid || q.UpperLimit.IsValid {
+		sbQuery.WriteString(" WHERE ")
+
+		if q.LowerLimit.IsValid {
+			fmt.Fprintf(&sbQuery, "[%s]", q.PrimaryKey)
+			if q.LowerLimit.IsInclusive {
+				sbQuery.WriteString(" >=")
+			} else {
+				sbQuery.WriteString(" >")
+			}
+			sbQuery.WriteString(" @min")
+		}
+
+		if q.LowerLimit.IsValid && q.UpperLimit.IsValid {
+			sbQuery.WriteString(" AND ")
+		}
+
+		if q.UpperLimit.IsValid {
+			fmt.Fprintf(&sbQuery, "[%s]", q.PrimaryKey)
+			if q.UpperLimit.IsInclusive {
+				sbQuery.WriteString(" <=")
+			} else {
+				sbQuery.WriteString(" <")
+			}
+			sbQuery.WriteString(" @max")
+		}
+	}
+
+	fmt.Fprintf(&sbQuery, " ORDER BY [%s] ASC", q.PrimaryKey)
+
+	queryString := sbQuery.String()
+
+	var queryArgs []any
+
+	if q.LowerLimit.IsValid {
+		queryArgs = append(queryArgs, sql.Named("min", q.LowerLimit.Value))
+	}
+
+	if q.UpperLimit.IsValid {
+		queryArgs = append(queryArgs, sql.Named("max", q.UpperLimit.Value))
+	}
+
+	return mw.Query(ctx, queryString, queryArgs...)
+}

internal/app/db-wrapper/postgres.go

@@ -3,14 +3,17 @@ package dbwrapper
 import (
 	"context"
 	"errors"
+	"fmt"
+	"strings"
 
+	dbdialects "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper/db_dialects"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 )
 
 func init() {
-	Register("postgres", func() DbWrapper {
+	Register(dbdialects.Postgres, func() DbWrapper {
-		return &postgresDbWrapper{dialect: "postgres"}
+		return &postgresDbWrapper{dialect: dbdialects.Postgres}
 	})
 }
 
@@ -126,3 +129,75 @@ func (pw *postgresDbWrapper) SaveMassive(ctx context.Context, schema string, tab
 
 	return affectedRows, nil
 }
+
+func (pw *postgresDbWrapper) QueryFromObject(ctx context.Context, q ExtractionQuery) (RowsResult, error) {
+	var sbQuery strings.Builder
+
+	sbQuery.WriteString("SELECT ")
+
+	if len(q.columns) == 0 {
+		sbQuery.WriteString("*")
+	} else {
+		for i, col := range q.columns {
+			switch col.Type() {
+			case "GEOMETRY":
+				fmt.Fprintf(&sbQuery, `ST_AsEWKB("%s") AS "%s"`, col.Name(), col.Name())
+			default:
+				fmt.Fprintf(&sbQuery, `"%s"`, col.Name())
+			}
+
+			if i < len(q.columns)-1 {
+				sbQuery.WriteString(", ")
+			}
+		}
+	}
+
+	fmt.Fprintf(&sbQuery, ` FROM "%s"."%s"`, q.Schema, q.Table)
+
+	if q.LowerLimit.IsValid || q.UpperLimit.IsValid {
+		sbQuery.WriteString(" WHERE ")
+		paramIdx := 1
+
+		if q.LowerLimit.IsValid {
+			fmt.Fprintf(&sbQuery, `"%s"`, q.PrimaryKey)
+			if q.LowerLimit.IsInclusive {
+				sbQuery.WriteString(" >=")
+			} else {
+				sbQuery.WriteString(" >")
+			}
+			fmt.Fprintf(&sbQuery, " $%d", paramIdx)
+			paramIdx++
+		}
+
+		if q.LowerLimit.IsValid && q.UpperLimit.IsValid {
+			sbQuery.WriteString(" AND ")
+		}
+
+		if q.UpperLimit.IsValid {
+			fmt.Fprintf(&sbQuery, `"%s"`, q.PrimaryKey)
+			if q.UpperLimit.IsInclusive {
+				sbQuery.WriteString(" <=")
+			} else {
+				sbQuery.WriteString(" <")
+			}
+			fmt.Fprintf(&sbQuery, " $%d", paramIdx)
+			paramIdx++
+		}
+	}
+
+	fmt.Fprintf(&sbQuery, ` ORDER BY "%s" ASC`, q.PrimaryKey)
+
+	queryString := sbQuery.String()
+
+	var queryArgs []any
+
+	if q.LowerLimit.IsValid {
+		queryArgs = append(queryArgs, q.LowerLimit.Value)
+	}
+
+	if q.UpperLimit.IsValid {
+		queryArgs = append(queryArgs, q.UpperLimit.Value)
+	}
+
+	return pw.Query(ctx, queryString, queryArgs...)
+}

internal/app/db-wrapper/types.go

@@ -3,6 +3,8 @@ package dbwrapper
 import (
 	"context"
 	"errors"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 )
 
 var MethodNotSupported error = errors.New("Method not supported by driver... yet :P")
@@ -24,6 +26,21 @@ type RowResult interface {
 	Scan(dest ...any) error
 }
 
+type ExtractorQueryLimit struct {
+	IsValid     bool
+	IsInclusive bool
+	Value       int64
+}
+
+type ExtractionQuery struct {
+	Schema     string
+	Table      string
+	PrimaryKey string
+	columns    []models.ColumnType
+	LowerLimit ExtractorQueryLimit
+	UpperLimit ExtractorQueryLimit
+}
+
 type DbWrapper interface {
 	Close() error
 	Connect(ctx context.Context, dbUrl string) error
@@ -32,4 +49,5 @@ type DbWrapper interface {
 	Query(ctx context.Context, query string, args ...any) (RowsResult, error)
 	QueryRow(ctx context.Context, query string, args ...any) RowResult
 	SaveMassive(ctx context.Context, schema string, table string, columnNames []string, rows [][]any) (int64, error)
+	QueryFromObject(ctx context.Context, query ExtractionQuery) (RowsResult, error)
 }

internal/app/etl/extractors/main.go

@@ -3,20 +3,153 @@ package extractors
 import (
 	"context"
 	"errors"
+	"fmt"
 	"slices"
 	"strings"
 	"sync"
 	"sync/atomic"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/convert"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
+	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/google/uuid"
 )
 
-func Consume(
+type GenericExtractor struct {
+	db dbwrapper.DbWrapper
+}
+
+func NewExtractor(db dbwrapper.DbWrapper) GenericExtractor {
+	return GenericExtractor{db: db}
+}
+
+func errorFromLastRow(
+	lastRow models.UnknownRowValues,
+	indexPrimaryKey int,
+	partition models.Partition,
+	previousError error,
+) error {
+	lastIdRawValue := lastRow[indexPrimaryKey]
+
+	lastId, ok := convert.ToInt64(lastIdRawValue)
+	if !ok {
+		currentPartition := partition
+		currentPartition.RetryCounter = 3
+		return &custom_errors.ExtractorError{
+			Partition: currentPartition,
+			HasLastId: true,
+			Msg:       fmt.Sprintf("Couldn't cast last id value as int: %s", previousError.Error()),
+		}
+	}
+
+	return &custom_errors.ExtractorError{
+		Partition: partition,
+		HasLastId: true,
+		LastId:    lastId,
+		Msg:       previousError.Error(),
+	}
+}
+
+func (ex *GenericExtractor) ProcessPartition(
+	ctx context.Context,
+	tableInfo config.SourceTableInfo,
+	columns []models.ColumnType,
+	batchSize int,
+	partition models.Partition,
+	indexPrimaryKey int,
+	chBatchesOut chan<- models.Batch,
+) (int, error) {
+	rowsRead := 0
+	query := dbwrapper.ExtractionQuery{
+		Schema:     tableInfo.Schema,
+		Table:      tableInfo.Table,
+		PrimaryKey: tableInfo.PrimaryKey,
+		LowerLimit: dbwrapper.ExtractorQueryLimit{
+			IsValid:     partition.HasRange && partition.Range.Min > 0,
+			IsInclusive: partition.Range.IsMinInclusive,
+			Value:       partition.Range.Min,
+		},
+		UpperLimit: dbwrapper.ExtractorQueryLimit{
+			IsValid:     partition.HasRange && partition.Range.Max > 0,
+			IsInclusive: partition.Range.IsMaxInclusive,
+			Value:       partition.Range.Max,
+		},
+	}
+
+	rows, err := ex.db.QueryFromObject(ctx, query)
+
+	if err != nil {
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+	}
+	defer rows.Close()
+
+	batchRows := make([]models.UnknownRowValues, 0, batchSize)
+
+	for rows.Next() {
+		rowValues := make([]any, len(columns))
+		scanArgs := make([]any, len(columns))
+
+		for i := range rowValues {
+			scanArgs[i] = &rowValues[i]
+		}
+
+		if err := rows.Scan(scanArgs...); err != nil {
+			if len(batchRows) == 0 {
+				return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+			}
+
+			lastRow := batchRows[len(batchRows)-1]
+
+			select {
+			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+			case <-ctx.Done():
+				return rowsRead, ctx.Err()
+			}
+
+			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
+		}
+		rowsRead++
+
+		batchRows = append(batchRows, rowValues)
+		if len(batchRows) >= batchSize {
+			select {
+			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+			case <-ctx.Done():
+				return rowsRead, ctx.Err()
+			}
+
+			batchRows = make([]models.UnknownRowValues, 0, batchSize)
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		if errors.Is(err, ctx.Err()) {
+			return rowsRead, ctx.Err()
+		}
+
+		if len(batchRows) > 0 {
+			lastRow := batchRows[len(batchRows)-1]
+			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
+		}
+
+		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
+	}
+
+	if len(batchRows) > 0 {
+		select {
+		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
+		case <-ctx.Done():
+			return rowsRead, ctx.Err()
+		}
+	}
+
+	return rowsRead, nil
+}
+
+func (ex *GenericExtractor) Consume(
 	ctx context.Context,
-	extractor etl.Extractor,
 	tableInfo config.SourceTableInfo,
 	columns []models.ColumnType,
 	batchSize int,
@@ -57,7 +190,7 @@ func Consume(
 				return
 			}
 
-			rowsReadResult, err := extractor.Exec(
+			rowsReadResult, err := ex.ProcessPartition(
 				ctx,
 				tableInfo,
 				columns,

internal/app/etl/extractors/mssql.go

@@ -1,209 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"strings"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/convert"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-type MssqlExtractor struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewMssqlExtractor(db dbwrapper.DbWrapper) etl.Extractor {
-	return &MssqlExtractor{db: db}
-}
-
-func buildExtractQueryMssql(
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	includeRange bool,
-	isMinInclusive bool,
-	isMaxInclusive bool,
-	hasMin bool,
-	hasMax bool,
-) string {
-	var sbQuery strings.Builder
-
-	sbQuery.WriteString("SELECT ")
-
-	if len(columns) == 0 {
-		sbQuery.WriteString("*")
-	} else {
-		for i, col := range columns {
-			fmt.Fprintf(&sbQuery, "[%s]", col.Name())
-
-			if col.Type() == "GEOMETRY" {
-				fmt.Fprintf(&sbQuery, ".STAsBinary() AS [%s]", col.Name())
-			}
-
-			if i < len(columns)-1 {
-				sbQuery.WriteString(", ")
-			}
-		}
-	}
-
-	fmt.Fprintf(&sbQuery, " FROM [%s].[%s]", tableInfo.Schema, tableInfo.Table)
-
-	if includeRange && (hasMin || hasMax) {
-		sbQuery.WriteString(" WHERE ")
-
-		if hasMin {
-			fmt.Fprintf(&sbQuery, "[%s]", tableInfo.PrimaryKey)
-			if isMinInclusive {
-				sbQuery.WriteString(" >=")
-			} else {
-				sbQuery.WriteString(" >")
-			}
-			sbQuery.WriteString(" @min")
-		}
-
-		if hasMin && hasMax {
-			sbQuery.WriteString(" AND ")
-		}
-
-		if hasMax {
-			fmt.Fprintf(&sbQuery, "[%s]", tableInfo.PrimaryKey)
-			if isMaxInclusive {
-				sbQuery.WriteString(" <=")
-			} else {
-				sbQuery.WriteString(" <")
-			}
-			sbQuery.WriteString(" @max")
-		}
-	}
-
-	fmt.Fprintf(&sbQuery, " ORDER BY [%s] ASC", tableInfo.PrimaryKey)
-
-	return sbQuery.String()
-}
-
-func errorFromLastRow(
-	lastRow models.UnknownRowValues,
-	indexPrimaryKey int,
-	partition models.Partition,
-	previousError error,
-) *custom_errors.ExtractorError {
-	lastIdRawValue := lastRow[indexPrimaryKey]
-
-	lastId, ok := convert.ToInt64(lastIdRawValue)
-	if !ok {
-		currentPartition := partition
-		currentPartition.RetryCounter = 3
-		return &custom_errors.ExtractorError{
-			Partition: currentPartition,
-			HasLastId: true,
-			Msg:       fmt.Sprintf("Couldn't cast last id value as int: %s", previousError.Error()),
-		}
-
-	}
-
-	return &custom_errors.ExtractorError{
-		Partition: partition,
-		HasLastId: true,
-		LastId:    lastId,
-		Msg:       previousError.Error(),
-	}
-}
-
-func (mssqlEx *MssqlExtractor) Exec(
-	ctx context.Context,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	partition models.Partition,
-	indexPrimaryKey int,
-	chBatchesOut chan<- models.Batch,
-) (int, error) {
-	hasMin := partition.HasRange && partition.Range.Min > 0
-	hasMax := partition.HasRange && partition.Range.Max > 0
-	query := buildExtractQueryMssql(tableInfo, columns, partition.HasRange, partition.Range.IsMinInclusive, partition.Range.IsMaxInclusive, hasMin, hasMax)
-
-	var queryArgs []any
-	if hasMin {
-		queryArgs = append(queryArgs, sql.Named("min", partition.Range.Min))
-	}
-	if hasMax {
-		queryArgs = append(queryArgs, sql.Named("max", partition.Range.Max))
-	}
-
-	rowsRead := 0
-	rows, err := mssqlEx.db.Query(ctx, query, queryArgs...)
-	if err != nil {
-		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
-	}
-	defer rows.Close()
-
-	batchRows := make([]models.UnknownRowValues, 0, batchSize)
-
-	for rows.Next() {
-		rowValues := make([]any, len(columns))
-		scanArgs := make([]any, len(columns))
-
-		for i := range rowValues {
-			scanArgs[i] = &rowValues[i]
-		}
-
-		if err := rows.Scan(scanArgs...); err != nil {
-			if len(batchRows) == 0 {
-				return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
-			}
-
-			lastRow := batchRows[len(batchRows)-1]
-
-			select {
-			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
-			case <-ctx.Done():
-				return rowsRead, ctx.Err()
-			}
-
-			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
-		}
-		rowsRead++
-
-		batchRows = append(batchRows, rowValues)
-		if len(batchRows) >= batchSize {
-			select {
-			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
-			case <-ctx.Done():
-				return rowsRead, ctx.Err()
-			}
-
-			batchRows = make([]models.UnknownRowValues, 0, batchSize)
-		}
-
-	}
-
-	if err := rows.Err(); err != nil {
-		if errors.Is(err, ctx.Err()) {
-			return rowsRead, ctx.Err()
-		}
-
-		if len(batchRows) > 0 {
-			lastRow := batchRows[len(batchRows)-1]
-			return rowsRead, errorFromLastRow(lastRow, indexPrimaryKey, partition, err)
-		}
-
-		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
-	}
-
-	if len(batchRows) > 0 {
-		select {
-		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
-		case <-ctx.Done():
-			return rowsRead, ctx.Err()
-		}
-	}
-
-	return rowsRead, nil
-}

internal/app/etl/extractors/postgres.go

@@ -1,157 +0,0 @@
-package extractors
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	dbwrapper "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/db-wrapper"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/google/uuid"
-)
-
-type PostgresExtractor struct {
-	db dbwrapper.DbWrapper
-}
-
-func NewPostgresExtractor(db dbwrapper.DbWrapper) etl.Extractor {
-	return &PostgresExtractor{db: db}
-}
-
-func buildExtractQueryPostgres(
-	sourceDbInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	includeRange bool,
-	isMinInclusive bool,
-	isMaxInclusive bool,
-	hasMin bool,
-	hasMax bool,
-) string {
-	var sbColumns strings.Builder
-
-	if len(columns) == 0 {
-		sbColumns.WriteString("*")
-	} else {
-		for i, col := range columns {
-			if col.Type() == "GEOMETRY" {
-				sbColumns.WriteString(`ST_AsEWKB("`)
-				sbColumns.WriteString(col.Name())
-				sbColumns.WriteString(`") AS "`)
-				sbColumns.WriteString(col.Name())
-				sbColumns.WriteString(`"`)
-			} else {
-				sbColumns.WriteString(`"`)
-				sbColumns.WriteString(col.Name())
-				sbColumns.WriteString(`"`)
-			}
-
-			if i < len(columns)-1 {
-				sbColumns.WriteString(", ")
-			}
-		}
-	}
-
-	query := fmt.Sprintf(`SELECT %s FROM "%s"."%s"`, sbColumns.String(), sourceDbInfo.Schema, sourceDbInfo.Table)
-
-	if includeRange && (hasMin || hasMax) {
-		query += " WHERE "
-		paramIdx := 1
-
-		if hasMin {
-			query += fmt.Sprintf(`"%s"`, sourceDbInfo.PrimaryKey)
-			if isMinInclusive {
-				query += " >="
-			} else {
-				query += " >"
-			}
-			query += fmt.Sprintf(" $%d", paramIdx)
-			paramIdx++
-		}
-
-		if hasMin && hasMax {
-			query += " AND "
-		}
-
-		if hasMax {
-			query += fmt.Sprintf(`"%s"`, sourceDbInfo.PrimaryKey)
-			if isMaxInclusive {
-				query += " <="
-			} else {
-				query += " <"
-			}
-			query += fmt.Sprintf(" $%d", paramIdx)
-		}
-	}
-
-	query += fmt.Sprintf(` ORDER BY "%s" ASC`, sourceDbInfo.PrimaryKey)
-
-	return query
-}
-
-func (postgresEx *PostgresExtractor) Exec(
-	ctx context.Context,
-	tableInfo config.SourceTableInfo,
-	columns []models.ColumnType,
-	batchSize int,
-	partition models.Partition,
-	indexPrimaryKey int,
-	chBatchesOut chan<- models.Batch,
-) (int, error) {
-	hasMin := partition.HasRange && partition.Range.Min > 0
-	hasMax := partition.HasRange && partition.Range.Max > 0
-	query := buildExtractQueryPostgres(tableInfo, columns, partition.HasRange, partition.Range.IsMinInclusive, partition.Range.IsMaxInclusive, hasMin, hasMax)
-
-	var queryArgs []any
-	if hasMin {
-		queryArgs = append(queryArgs, partition.Range.Min)
-	}
-	if hasMax {
-		queryArgs = append(queryArgs, partition.Range.Max)
-	}
-
-	rowsRead := 0
-	rows, err := postgresEx.db.Query(ctx, query, queryArgs...)
-	if err != nil {
-		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
-	}
-	defer rows.Close()
-
-	batchRows := make([]models.UnknownRowValues, 0, batchSize)
-
-	for rows.Next() {
-		values, err := rows.Values()
-		if err != nil {
-			return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
-		}
-		rowsRead++
-
-		batchRows = append(batchRows, values)
-
-		if len(batchRows) >= batchSize {
-			select {
-			case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
-			case <-ctx.Done():
-				return rowsRead, ctx.Err()
-			}
-
-			batchRows = make([]models.UnknownRowValues, 0, batchSize)
-		}
-	}
-
-	if err := rows.Err(); err != nil {
-		return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
-	}
-
-	if len(batchRows) > 0 {
-		select {
-		case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
-		case <-ctx.Done():
-			return rowsRead, nil
-		}
-	}
-
-	return rowsRead, nil
-}

internal/app/etl/transformers/mssql.go

@@ -119,11 +119,13 @@ func computeStorageTransformationPlan(
 						schema, table, sourceColName, v)
 					return v, nil
 				}
-				blobPath := fmt.Sprintf("%s/%s/%s/%s.bin", schema, table, sourceColName, uuid.New().String())
+				start := time.Now()
+				blobPath := fmt.Sprintf("%s/%s/%s", schema, table, uuid.New().String())
 				blobURL, err := azureClient.UploadAndGetURL(ctx, blobPath, b)
 				if err != nil {
 					return nil, fmt.Errorf("uploading %s.%s.%s: %w", schema, table, sourceColName, err)
 				}
+				log.Debugf(`Succesfully uploaded "%s", (%vms)`, blobURL, time.Since(start).Milliseconds())
 				return blobURL, nil
 			},
 		})

internal/app/etl/types.go

@@ -10,7 +10,7 @@ import (
 )
 
 type Extractor interface {
-	Exec(
+	ProcessPartition(
 		ctx context.Context,
 		tableInfo config.SourceTableInfo,
 		columns []models.ColumnType,