feat: add MSSQL extractor and transformer implementations for improved data migration
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractor"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformer"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
@@ -60,6 +61,9 @@ func processMigrationJob(
|
||||
var wgTransformers sync.WaitGroup
|
||||
var wgLoaders sync.WaitGroup
|
||||
|
||||
mssqlExtractor := extractor.NewMssqlExtractor(sourceDb)
|
||||
mssqlToPostgresTransformer := transformer.NewMssqlTransformer()
|
||||
|
||||
go func() {
|
||||
if err := custom_errors.JobErrorHandler(jobCtx, chJobErrors); err != nil {
|
||||
cancel()
|
||||
@@ -73,11 +77,9 @@ func processMigrationJob(
|
||||
maxExtractors := min(job.MaxExtractors, len(batches))
|
||||
log.Infof("Starting %d extractor(s)...", maxExtractors)
|
||||
|
||||
exMssql := extractor.NewMssqlExtractor(sourceDb)
|
||||
|
||||
for range maxExtractors {
|
||||
wgExtractors.Go(func() {
|
||||
exMssql.Exec(
|
||||
mssqlExtractor.Exec(
|
||||
jobCtx,
|
||||
job.SourceTable,
|
||||
sourceColTypes,
|
||||
@@ -103,7 +105,14 @@ func processMigrationJob(
|
||||
|
||||
for range maxExtractors {
|
||||
wgTransformers.Go(func() {
|
||||
transformRowsMssql(jobCtx, sourceColTypes, chChunksRaw, chChunksTransformed, chJobErrors, &wgActiveChunks)
|
||||
mssqlToPostgresTransformer.Exec(
|
||||
jobCtx,
|
||||
sourceColTypes,
|
||||
chChunksRaw,
|
||||
chChunksTransformed,
|
||||
chJobErrors,
|
||||
&wgActiveChunks,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user