feat: implement MSSQL extractor, transformer, and Postgres loader for enhanced data migration
This commit is contained in:
@@ -9,12 +9,9 @@ import (
|
||||
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/extractor"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/loader"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl/transformer"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
_ "github.com/microsoft/go-mssqldb"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -23,6 +20,9 @@ func processMigrationJob(
|
||||
ctx context.Context,
|
||||
sourceDb *sql.DB,
|
||||
targetDb *pgxpool.Pool,
|
||||
extractor etl.Extractor,
|
||||
transformer etl.Transformer,
|
||||
loader etl.Loader,
|
||||
job config.Job,
|
||||
) JobResult {
|
||||
result := JobResult{
|
||||
@@ -62,10 +62,6 @@ func processMigrationJob(
|
||||
var wgTransformers sync.WaitGroup
|
||||
var wgLoaders sync.WaitGroup
|
||||
|
||||
mssqlExtractor := extractor.NewMssqlExtractor(sourceDb)
|
||||
mssqlToPostgresTransformer := transformer.NewMssqlTransformer()
|
||||
postgresLoader := loader.NewPostgresLoader(targetDb)
|
||||
|
||||
go func() {
|
||||
if err := custom_errors.JobErrorHandler(jobCtx, chJobErrors); err != nil {
|
||||
cancel()
|
||||
@@ -81,7 +77,7 @@ func processMigrationJob(
|
||||
|
||||
for range maxExtractors {
|
||||
wgExtractors.Go(func() {
|
||||
mssqlExtractor.Exec(
|
||||
extractor.Exec(
|
||||
jobCtx,
|
||||
job.SourceTable,
|
||||
sourceColTypes,
|
||||
@@ -107,7 +103,7 @@ func processMigrationJob(
|
||||
|
||||
for range maxExtractors {
|
||||
wgTransformers.Go(func() {
|
||||
mssqlToPostgresTransformer.Exec(
|
||||
transformer.Exec(
|
||||
jobCtx,
|
||||
sourceColTypes,
|
||||
chChunksRaw,
|
||||
@@ -122,7 +118,7 @@ func processMigrationJob(
|
||||
|
||||
for range job.MaxLoaders {
|
||||
wgLoaders.Go(func() {
|
||||
postgresLoader.Exec(
|
||||
loader.Exec(
|
||||
jobCtx,
|
||||
job.TargetTable,
|
||||
targetColTypes,
|
||||
|
||||
Reference in New Issue
Block a user