feat: implement MSSQL extractor, transformer, and Postgres loader for enhanced data migration

internal/app/etl/loaders/postgres.go

@@ -0,0 +1,124 @@
+package loaders
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+type PostgresLoader struct {
+	db *pgxpool.Pool
+}
+
+func NewPostgresLoader(pool *pgxpool.Pool) etl.Loader {
+	return &PostgresLoader{db: pool}
+}
+
+func mapSlice[T any, V any](input []T, mapper func(T) V) []V {
+	result := make([]V, len(input))
+
+	for i, v := range input {
+		result[i] = mapper(v)
+	}
+
+	return result
+}
+
+func (postgresLd *PostgresLoader) ProcessChunk(
+	ctx context.Context,
+	tableInfo config.TargetTableInfo,
+	colNames []string,
+	chunk models.Chunk,
+) (int, error) {
+	tableId := pgx.Identifier{tableInfo.Schema, tableInfo.Table}
+	_, err := postgresLd.db.CopyFrom(
+		ctx,
+		tableId,
+		colNames,
+		pgx.CopyFromRows(chunk.Data),
+	)
+
+	if err != nil {
+		var pgErr *pgconn.PgError
+		if errors.As(err, &pgErr) {
+			if pgErr.Code == "23505" {
+				return 0, &custom_errors.JobError{
+					ShouldCancelJob: true,
+					Msg:             fmt.Sprintf("Fatal error in table %s", tableId.Sanitize()),
+					Prev:            err,
+				}
+			}
+		}
+
+		return 0, &custom_errors.LoaderError{Chunk: chunk, Msg: err.Error()}
+	}
+
+	return len(chunk.Data), nil
+}
+
+func (postgresLd *PostgresLoader) Exec(
+	ctx context.Context,
+	tableInfo config.TargetTableInfo,
+	columns []models.ColumnType,
+	chChunksIn <-chan models.Chunk,
+	chErrorsOut chan<- custom_errors.LoaderError,
+	chJobErrorsOut chan<- custom_errors.JobError,
+	wgActiveChunks *sync.WaitGroup,
+	rowsLoaded *int64,
+) {
+	colNames := mapSlice(columns, func(col models.ColumnType) string {
+		return col.Name()
+	})
+
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case chunk, ok := <-chChunksIn:
+			if !ok {
+				return
+			}
+
+			processedRows, err := postgresLd.ProcessChunk(ctx, tableInfo, colNames, chunk)
+
+			if err != nil {
+				var ldError *custom_errors.LoaderError
+				if errors.As(err, &ldError) {
+					select {
+					case <-ctx.Done():
+						return
+					case chErrorsOut <- *ldError:
+					}
+				}
+
+				var jobError *custom_errors.JobError
+				if errors.As(err, &jobError) {
+					select {
+					case <-ctx.Done():
+						return
+					case chJobErrorsOut <- *jobError:
+					}
+				}
+
+				return
+			}
+
+			wgActiveChunks.Done()
+			atomic.AddInt64(rowsLoaded, int64(processedRows))
+		}
+	}
+}