kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: refactor migration job structure to use SourceTableInfo and TargetTableInfo for improved configuration handling

Browse Source
This commit is contained in:
Kylesoda
2026-04-09 19:20:50 -05:00
parent e8ace6ecf9
commit 524d892a60
9 changed files with 118 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
cmd/go_migrate/extractor.go
View File

@@ -9,6 +9,7 @@ import (
"sync"
"time"
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
_ "github.com/microsoft/go-mssqldb"
@@ -27,7 +28,7 @@ type Chunk struct {
func extractFromMssql(
ctx context.Context,
db *sql.DB,
job MigrationJob,
tableInfo config.SourceTableInfo,
columns []ColumnType,
chunkSize int,
chBatchesIn <-chan Batch,
@@ -37,7 +38,7 @@ func extractFromMssql(
wgActiveBatches *sync.WaitGroup,
) {
indexPrimaryKey := slices.IndexFunc(columns, func(col ColumnType) bool {
return strings.EqualFold(col.name, job.PrimaryKey)
return strings.EqualFold(col.name, tableInfo.PrimaryKey)
})
if indexPrimaryKey == -1 {
@@ -68,7 +69,7 @@ func extractFromMssql(
return
}
if abort := processBatch(ctx, db, job, columns, chunkSize, batch, indexPrimaryKey, chChunksOut, chErrorsOut, wgActiveBatches); abort {
if abort := processBatch(ctx, db, tableInfo, columns, chunkSize, batch, indexPrimaryKey, chChunksOut, chErrorsOut, wgActiveBatches); abort {
return
}
}
@@ -78,7 +79,7 @@ func extractFromMssql(
func processBatch(
ctx context.Context,
db *sql.DB,
job MigrationJob,
tableInfo config.SourceTableInfo,
columns []ColumnType,
chunkSize int,
batch Batch,
@@ -87,7 +88,7 @@ func processBatch(
chErrorsOut chan<- ExtractorError,
wgActiveBatches *sync.WaitGroup,
) (abort bool) {
query := buildExtractQueryMssql(job, columns, batch.ShouldUseRange, batch.IsLowerLimitInclusive)
query := buildExtractQueryMssql(tableInfo, columns, batch.ShouldUseRange, batch.IsLowerLimitInclusive)
log.Debug("Query used to extract data from mssql: ", query)
var queryArgs []any
@@ -206,8 +207,8 @@ func processBatch(
return false
}
func extractFromPostgres(ctx context.Context, job MigrationJob, columns []ColumnType, chunkSize int, db *pgxpool.Pool, out chan<- []UnknownRowValues) error {
query := buildExtractQueryPostgres(job, columns)
func extractFromPostgres(ctx context.Context, tableInfo config.SourceTableInfo, columns []ColumnType, chunkSize int, db *pgxpool.Pool, out chan<- []UnknownRowValues) error {
query := buildExtractQueryPostgres(tableInfo, columns)
log.Debug("Query used to extract data from postgres: ", query)
rows, err := db.Query(ctx, query)
@@ -229,13 +230,13 @@ func extractFromPostgres(ctx context.Context, job MigrationJob, columns []Column
if len(rowsChunk) >= chunkSize {
out <- rowsChunk
rowsChunk = make([]UnknownRowValues, 0, chunkSize)
log.Infof("Chunk send... %+v", job)
log.Infof("Chunk send... %+v", tableInfo)
}
}
if len(rowsChunk) > 0 {
out <- rowsChunk
log.Infof("Chunk send... %+v", job)
log.Infof("Chunk send... %+v", tableInfo)
}
return nil