kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update chunk size for MSSQL processing and enhance error handling in transformation functions

Browse Source
This commit is contained in:
Kylesoda
2026-04-08 20:48:36 -05:00
parent eeef3bc813
commit 853be4a5a6
4 changed files with 69 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/go_migrate/process.go
View File

@@ -74,20 +74,20 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
log.Infof("Extraction completed in %v", time.Since(extractStartTime))
}()
chRowsTransform := make(chan []UnknownRowValues, QueueSize)
chChunksTransform := make(chan []UnknownRowValues, QueueSize)
var wgMssqlTransformers sync.WaitGroup
log.Infof("Starting %d MSSQL transformers...", maxExtractors)
transformStartTime := time.Now()
for range maxExtractors {
wgMssqlTransformers.Go(func() {
transformRowsMssql(sourceColTypes, chChunks, chRowsTransform)
transformRowsMssql(sourceColTypes, chChunks, chChunksTransform, chJobErrors)
})
}
go func() {
wgMssqlTransformers.Wait()
close(chRowsTransform)
close(chChunksTransform)
log.Infof("Transformation completed in %v", time.Since(transformStartTime))
}()
@@ -98,7 +98,7 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
for range NumLoaders {
wgPostgresLoaders.Go(func() {
if err := loadRowsPostgres(ctx, job, targetColTypes, targetDb, chRowsTransform); err != nil {
if err := loadRowsPostgres(ctx, job, targetColTypes, targetDb, chChunksTransform); err != nil {
log.Error("Unexpected error loading data into postgres: ", err)
}
})