feat: enhance migration job processing with detailed metrics and error handling

cmd/go_migrate/process.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
@@ -18,13 +19,18 @@ func processMigrationJob(
 	sourceDb *sql.DB,
 	targetDb *pgxpool.Pool,
 	job config.Job,
-) {
-	jobStartTime := time.Now()
-	log.Infof("Starting migration job: %s.%s [PK: %s]", job.SourceTable.Schema, job.SourceTable.Table, job.SourceTable.PrimaryKey)
+) JobResult {
+	result := JobResult{
+		JobName:   job.Name,
+		StartTime: time.Now(),
+	}
+
+	var rowsRead, rowsLoaded, rowsFailed int64
 
 	sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job.SourceTable, job.TargetTable)
 	if err != nil {
-		log.Fatal("Unexpected error: ", err)
+		result.Error = err
+		return result
 	}
 
 	logColumnTypes(sourceColTypes, "Source col types")
@@ -54,6 +60,7 @@ func processMigrationJob(
 	go func() {
 		if err := jobErrorHandler(jobCtx, chJobErrors); err != nil {
 			cancel()
+			result.Error = err
 		}
 	}()
 
@@ -62,11 +69,10 @@ func processMigrationJob(
 
 	maxExtractors := min(job.MaxExtractors, len(batches))
 	log.Infof("Starting %d extractor(s)...", maxExtractors)
-	extractStartTime := time.Now()
 
 	for range maxExtractors {
 		wgExtractors.Go(func() {
-			extractFromMssql(jobCtx, sourceDb, job.SourceTable, sourceColTypes, job.ChunkSize, chBatches, chChunksRaw, chExtractorErrors, chJobErrors, &wgActiveBatches)
+			extractFromMssql(jobCtx, sourceDb, job.SourceTable, sourceColTypes, job.ChunkSize, chBatches, chChunksRaw, chExtractorErrors, chJobErrors, &wgActiveBatches, &rowsRead)
 		})
 	}
 
@@ -78,7 +84,6 @@ func processMigrationJob(
 	}()
 
 	log.Infof("Starting %d transformer(s)...", maxExtractors)
-	transformStartTime := time.Now()
 
 	for range maxExtractors {
 		wgTransformers.Go(func() {
@@ -87,11 +92,10 @@ func processMigrationJob(
 	}
 
 	log.Infof("Starting %d loader(s)...", job.MaxLoaders)
-	loadStartTime := time.Now()
 
 	for range job.MaxLoaders {
 		wgLoaders.Go(func() {
-			loadRowsPostgres(jobCtx, targetDb, job.TargetTable, targetColTypes, chChunksTransformed, chLoadersErrors, chJobErrors, &wgActiveChunks)
+			loadRowsPostgres(jobCtx, targetDb, job.TargetTable, targetColTypes, chChunksTransformed, chLoadersErrors, chJobErrors, &wgActiveChunks, &rowsLoaded)
 		})
 	}
 
@@ -101,24 +105,31 @@ func processMigrationJob(
 		close(chExtractorErrors)
 
 		wgExtractors.Wait()
-		log.Infof("Extraction completed in %v", time.Since(extractStartTime))
 		close(chChunksRaw)
 
 		wgTransformers.Wait()
-		log.Infof("Transformation completed in %v", time.Since(transformStartTime))
 
 		wgActiveChunks.Wait()
 		close(chChunksTransformed)
 		close(chLoadersErrors)
 
 		wgLoaders.Wait()
-		log.Infof("Loading completed in %v", time.Since(loadStartTime))
 
 		cancel()
 	}()
 
 	<-jobCtx.Done()
-	log.Infof("Migration job completed (%s.%s). Total time: %v", job.SourceTable.Schema, job.SourceTable.Table, time.Since(jobStartTime))
+
+	if ctx.Err() != nil {
+		result.Error = ctx.Err()
+	}
+
+	result.Duration = time.Since(result.StartTime)
+	result.RowsRead = atomic.LoadInt64(&rowsRead)
+	result.RowsLoaded = atomic.LoadInt64(&rowsLoaded)
+	result.RowsFailed = atomic.LoadInt64(&rowsFailed)
+
+	return result
 }
 
 func logColumnTypes(columnTypes []ColumnType, label string) {