kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance logging and performance metrics for migration processes

Browse Source
This commit is contained in:
Kylesoda
2026-04-07 15:53:17 -05:00
parent 7bb67ddfcf
commit 270a66dbbf
6 changed files with 78 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/go_migrate/process.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"sync"
"time"
"github.com/jackc/pgx/v5/pgxpool"
@@ -12,6 +13,9 @@ import (
)
func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job MigrationJob) {
jobStartTime := time.Now()
log.Infof("Starting migration job: %s.%s [PK: %s]", job.Schema, job.Table, job.PrimaryKey)
sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job)
if err != nil {
log.Fatal("Unexpected error: ", err)
@@ -25,20 +29,29 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
mssqlCtx := context.Background()
go func() {
log.Info("Starting extraction from MSSQL...")
extractStartTime := time.Now()
if err := extractFromMssql(mssqlCtx, job, sourceColTypes, ChunkSize, sourceDb, chRowsExtract); err != nil {
log.Error("Unexpected error extrating data from mssql: ", err)
log.Error("Unexpected error extracting data from mssql: ", err)
}
close(chRowsExtract)
log.Infof("Extraction completed in %v", time.Since(extractStartTime))
}()
go func() {
log.Info("Starting transformation of rows...")
transformStartTime := time.Now()
transformRowsMssql(sourceColTypes, chRowsExtract, chRowsTransform)
close(chRowsTransform)
log.Infof("Transformation completed in %v", time.Since(transformStartTime))
}()
var wgPostgresLoaders sync.WaitGroup
postgresLoaderCtx := context.Background()
log.Infof("Starting %d PostgreSQL loader(s)...", NumLoaders)
loaderStartTime := time.Now()
for range NumLoaders {
wgPostgresLoaders.Go(func() {
if err := loadRowsPostgres(postgresLoaderCtx, job, sourceColTypes, targetDb, chRowsTransform); err != nil {
@@ -48,13 +61,17 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
}
wgPostgresLoaders.Wait()
log.Infof("Loading completed in %v", time.Since(loaderStartTime))
totalDuration := time.Since(jobStartTime)
log.Infof("Migration job completed successfully! Total time: %v", totalDuration)
}
func logColumnTypes(columnTypes []ColumnType, label string) {
log.Info(label)
log.Debug(label)
for _, col := range columnTypes {
log.Infof("%+v", col)
log.Debugf("%+v", col)
}
}