refactor: remove unused ColumnType and inspect-columns files; update migration job to use separate table analyzers for source and target databases

cmd/go_migrate/process.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -12,21 +11,22 @@ import (
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/table_analyzers"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
-	"github.com/jackc/pgx/v5/pgxpool"
-	_ "github.com/microsoft/go-mssqldb"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 )
 
 func processMigrationJob(
 	ctx context.Context,
-	sourceDb *sql.DB,
-	targetDb *pgxpool.Pool,
-	tableAnalyzer etl.TableAnalyzer,
+	sourceTableAnalyzer etl.TableAnalyzer,
+	targetTableAnalyzer etl.TableAnalyzer,
 	extractor etl.Extractor,
 	transformer etl.Transformer,
 	loader etl.Loader,
 	job config.Job,
 ) JobResult {
+	jobCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	result := JobResult{
 		JobName:   job.Name,
 		StartTime: time.Now(),
@@ -34,21 +34,38 @@ func processMigrationJob(
 
 	var rowsRead, rowsLoaded, rowsFailed int64
 
-	sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job.SourceTable, job.TargetTable)
+	var wgQueryColumnTypes errgroup.Group
+	var sourceColTypes, targetColTypes []models.ColumnType
+
+	wgQueryColumnTypes.Go(func() error {
+		var err error
+		sourceColTypes, err = sourceTableAnalyzer.QueryColumnTypes(jobCtx, job.SourceTable.TableInfo)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	wgQueryColumnTypes.Go(func() error {
+		var err error
+		targetColTypes, err = targetTableAnalyzer.QueryColumnTypes(jobCtx, job.TargetTable.TableInfo)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	err := wgQueryColumnTypes.Wait()
 	if err != nil {
 		result.Error = err
 		return result
 	}
 
-	logColumnTypes(sourceColTypes, "Source col types")
-	logColumnTypes(targetColTypes, "Target col types")
-
-	jobCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
 	partitions, err := table_analyzers.PartitionRangeGenerator(
 		jobCtx,
-		tableAnalyzer,
+		sourceTableAnalyzer,
 		job.SourceTable.TableInfo,
 		job.SourceTable.PrimaryKey,
 		job.RowsPerPartition,
@@ -72,6 +89,7 @@ func processMigrationJob(
 
 	go func() {
 		if err := custom_errors.JobErrorHandler(jobCtx, chJobErrors); err != nil {
+			log.Error("Fatal error received from JobErrorHandler, canceling job... - ", err)
 			cancel()
 			result.Error = err
 		}
@@ -140,25 +158,39 @@ func processMigrationJob(
 	}
 
 	go func() {
+		log.Debugf("Waiting for goroutines (%v)", job.Name)
+
 		wgActivePartitions.Wait()
+		log.Debugf("wgActivePartitions is empty (%v)", job.Name)
 		close(chPartitions)
+		log.Debugf("chPartitions is closed (%v)", job.Name)
 		close(chExtractorErrors)
+		log.Debugf("chExtractorErrors is closed (%v)", job.Name)
 
 		wgExtractors.Wait()
+		log.Debugf("wgExtractors is empty (%v)", job.Name)
 		close(chBatchesRaw)
+		log.Debugf("chBatchesRaw is closed (%v)", job.Name)
 
 		wgTransformers.Wait()
+		log.Debugf("wgTransformers is empty (%v)", job.Name)
 
 		wgActiveBatches.Wait()
+		log.Debugf("wgActiveBatches is empty (%v)", job.Name)
 		close(chBatchesTransformed)
+		log.Debugf("chBatchesTransformed is empty (%v)", job.Name)
 		close(chLoadersErrors)
+		log.Debugf("chLoadersErrors is empty (%v)", job.Name)
 
 		wgLoaders.Wait()
+		log.Debugf("wgLoaders is empty (%v)", job.Name)
 
 		cancel()
 	}()
 
+	log.Debugf("waiting for local context to be done (%v)", job.Name)
 	<-jobCtx.Done()
+	log.Debugf("local context done (%v)", job.Name)
 
 	if ctx.Err() != nil {
 		result.Error = ctx.Err()
@@ -171,11 +203,3 @@ func processMigrationJob(
 
 	return result
 }
-
-func logColumnTypes(columnTypes []models.ColumnType, label string) {
-	log.Debug(label)
-
-	for _, col := range columnTypes {
-		log.Debugf("%+v", col)
-	}
-}