feat: implement MSSQL extractor, transformer, and Postgres loader for enhanced data migration

cmd/go_migrate/main.go

@@ -7,6 +7,9 @@ import (
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractors"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loaders"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformers"
 	"github.com/jackc/pgx/v5/pgxpool"
 	log "github.com/sirupsen/logrus"
 )
@@ -87,11 +90,24 @@ func processMigrationJobs(
 	chJobs := make(chan config.Job, len(jobs))
 	var wgJobs sync.WaitGroup
 
+	extractor := extractors.NewMssqlExtractor(sourceDb)
+	transformer := transformers.NewMssqlTransformer()
+	loader := loaders.NewPostgresLoader(targetDb)
+
 	for i := range maxParallelWorkers {
 		wgJobs.Go(func() {
 			for job := range chJobs {
 				log.Infof("[worker %d] >>> Processing job: %s.%s <<<", i, job.SourceTable.Schema, job.SourceTable.Table)
-				res := processMigrationJob(ctx, sourceDb, targetDb, job)
+				res := processMigrationJob(
+					ctx,
+					sourceDb,
+					targetDb,
+					extractor,
+					transformer,
+					loader,
+					job,
+				)
+
 				chJobResults <- res
 			}
 		})

cmd/go_migrate/process.go

@@ -9,12 +9,9 @@ import (
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractor"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loader"
-	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformer"
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	"github.com/jackc/pgx/v5/pgxpool"
-
 	_ "github.com/microsoft/go-mssqldb"
 	log "github.com/sirupsen/logrus"
 )
@@ -23,6 +20,9 @@ func processMigrationJob(
 	ctx context.Context,
 	sourceDb *sql.DB,
 	targetDb *pgxpool.Pool,
+	extractor etl.Extractor,
+	transformer etl.Transformer,
+	loader etl.Loader,
 	job config.Job,
 ) JobResult {
 	result := JobResult{
@@ -62,10 +62,6 @@ func processMigrationJob(
 	var wgTransformers sync.WaitGroup
 	var wgLoaders sync.WaitGroup
 
-	mssqlExtractor := extractor.NewMssqlExtractor(sourceDb)
-	mssqlToPostgresTransformer := transformer.NewMssqlTransformer()
-	postgresLoader := loader.NewPostgresLoader(targetDb)
-
 	go func() {
 		if err := custom_errors.JobErrorHandler(jobCtx, chJobErrors); err != nil {
 			cancel()
@@ -81,7 +77,7 @@ func processMigrationJob(
 
 	for range maxExtractors {
 		wgExtractors.Go(func() {
-			mssqlExtractor.Exec(
+			extractor.Exec(
 				jobCtx,
 				job.SourceTable,
 				sourceColTypes,
@@ -107,7 +103,7 @@ func processMigrationJob(
 
 	for range maxExtractors {
 		wgTransformers.Go(func() {
-			mssqlToPostgresTransformer.Exec(
+			transformer.Exec(
 				jobCtx,
 				sourceColTypes,
 				chChunksRaw,
@@ -122,7 +118,7 @@ func processMigrationJob(
 
 	for range job.MaxLoaders {
 		wgLoaders.Go(func() {
-			postgresLoader.Exec(
+			loader.Exec(
 				jobCtx,
 				job.TargetTable,
 				targetColTypes,