kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: refactor migration job structure to use SourceTableInfo and TargetTableInfo for improved configuration handling

Browse Source
This commit is contained in:
Kylesoda
2026-04-09 19:20:50 -05:00
parent e8ace6ecf9
commit 524d892a60
9 changed files with 118 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/go_migrate/process.go
View File

@@ -6,6 +6,7 @@ import (
"sync"
"time"
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
"github.com/jackc/pgx/v5/pgxpool"
_ "github.com/microsoft/go-mssqldb"
@@ -16,12 +17,12 @@ func processMigrationJob(
ctx context.Context,
sourceDb *sql.DB,
targetDb *pgxpool.Pool,
job MigrationJob,
job config.Job,
) {
jobStartTime := time.Now()
log.Infof("Starting migration job: %s.%s [PK: %s]", job.Schema, job.Table, job.PrimaryKey)
log.Infof("Starting migration job: %s.%s [PK: %s]", job.SourceTable.Schema, job.SourceTable.Table, job.SourceTable.PrimaryKey)
sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job)
sourceColTypes, targetColTypes, err := GetColumnTypes(sourceDb, targetDb, job.SourceTable, job.TargetTable)
if err != nil {
log.Fatal("Unexpected error: ", err)
}
@@ -32,7 +33,7 @@ func processMigrationJob(
jobCtx, cancel := context.WithCancel(ctx)
defer cancel()
batches, err := batchGeneratorMssql(jobCtx, sourceDb, job)
batches, err := batchGeneratorMssql(jobCtx, sourceDb, job.SourceTable)
if err != nil {
log.Error("Unexpected error calculating batch ranges: ", err)
}
@@ -65,7 +66,7 @@ func processMigrationJob(
for range maxExtractors {
wgExtractors.Go(func() {
extractFromMssql(jobCtx, sourceDb, job, sourceColTypes, ChunkSize, chBatches, chChunksRaw, chExtractorErrors, chJobErrors, &wgActiveBatches)
extractFromMssql(jobCtx, sourceDb, job.SourceTable, sourceColTypes, ChunkSize, chBatches, chChunksRaw, chExtractorErrors, chJobErrors, &wgActiveBatches)
})
}
@@ -90,7 +91,7 @@ func processMigrationJob(
for range NumLoaders {
wgLoaders.Go(func() {
loadRowsPostgres(jobCtx, targetDb, job, targetColTypes, chChunksTransformed, chLoadersErrors, chJobErrors, &wgActiveChunks)
loadRowsPostgres(jobCtx, targetDb, job.TargetTable, targetColTypes, chChunksTransformed, chLoadersErrors, chJobErrors, &wgActiveChunks)
})
}
@@ -128,8 +129,14 @@ func logColumnTypes(columnTypes []ColumnType, label string) {
}
}
func logSampleRow(job MigrationJob, columns []ColumnType, rowValues UnknownRowValues, tag string) {
log.Infof("[%s.%s] Sample row: (%s)", job.Schema, job.Table, tag)
func logSampleRow(
schema string,
table string,
columns []ColumnType,
rowValues UnknownRowValues,
tag string,
) {
log.Infof("[%s.%s] Sample row: (%s)", schema, table, tag)
for i, col := range columns {
log.Infof("%s (%T): %v", col.Name(), rowValues[i], rowValues[i])
}