package main import ( "context" "database/sql" "sync" "time" "github.com/jackc/pgx/v5/pgxpool" _ "github.com/microsoft/go-mssqldb" log "github.com/sirupsen/logrus" ) func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job MigrationJob) { jobStartTime := time.Now() log.Infof("Starting migration job: %s.%s [PK: %s]", job.Schema, job.Table, job.PrimaryKey) sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job) if err != nil { log.Fatal("Unexpected error: ", err) } logColumnTypes(sourceColTypes, "Source col types") logColumnTypes(targetColTypes, "Target col types") mssqlCtx := context.Background() batchRanges, err := calculateBatchMetrics(mssqlCtx, sourceDb, job) if err != nil { log.Error("Unexpected error calculating batch ranges: ", err) } chBatchRanges := make(chan BatchRange, len(batchRanges)) maxExtractors := min(NumExtractors, len(batchRanges)) chRowsExtract := make(chan []UnknownRowValues, QueueSize) var wgMssqlExtractors sync.WaitGroup log.Infof("Starting %d MSSQL extractors...", maxExtractors) extractStartTime := time.Now() for range maxExtractors { wgMssqlExtractors.Go(func() { for br := range chBatchRanges { if err := extractFromMssql(mssqlCtx, sourceDb, job, sourceColTypes, ChunkSize, br, chRowsExtract); err != nil { log.Error("Unexpected error extracting data from mssql: ", err) } } }) } go func() { for _, br := range batchRanges { chBatchRanges <- br } close(chBatchRanges) }() go func() { wgMssqlExtractors.Wait() close(chRowsExtract) log.Infof("Extraction completed in %v", time.Since(extractStartTime)) }() chRowsTransform := make(chan []UnknownRowValues, QueueSize) var wgMssqlTransformers sync.WaitGroup log.Infof("Starting %d MSSQL transformers...", maxExtractors) transformStartTime := time.Now() for range maxExtractors { wgMssqlTransformers.Go(func() { transformRowsMssql(sourceColTypes, chRowsExtract, chRowsTransform) }) } go func() { wgMssqlTransformers.Wait() close(chRowsTransform) log.Infof("Transformation completed in %v", time.Since(transformStartTime)) }() var wgPostgresLoaders sync.WaitGroup // postgresLoaderCtx := context.Background() log.Infof("Starting %d PostgreSQL loader(s)...", NumLoaders) loaderStartTime := time.Now() for range NumLoaders { wgPostgresLoaders.Go(func() { // if err := loadRowsPostgres(postgresLoaderCtx, job, sourceColTypes, targetDb, chRowsTransform); err != nil { // log.Error("Unexpected error loading data into postgres: ", err) // } fakeLoader(job, sourceColTypes, chRowsTransform) }) } wgPostgresLoaders.Wait() log.Infof("Loading completed in %v", time.Since(loaderStartTime)) totalDuration := time.Since(jobStartTime) log.Infof("Migration job completed successfully! Total time: %v", totalDuration) } func logColumnTypes(columnTypes []ColumnType, label string) { log.Debug(label) for _, col := range columnTypes { log.Debugf("%+v", col) } } func logSampleRow(job MigrationJob, columns []ColumnType, rowValues UnknownRowValues, tag string) { log.Infof("[%s.%s] Sample row: (%s)", job.Schema, job.Table, tag) for i, col := range columns { log.Infof("%s (%T): %v", col.Name(), rowValues[i], rowValues[i]) } }