kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve error handling and job cancellation in migration process

Browse Source
This commit is contained in:
Kylesoda
2026-04-09 00:38:16 -05:00
parent 51480015ba
commit b64a76ca45
4 changed files with 31 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/go_migrate/process.go
View File

@@ -32,7 +32,7 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
log.Error("Unexpected error calculating batch ranges: ", err)
}
chJobErrors := make(chan JobError, 100)
chJobErrors := make(chan JobError, 50)
chBatches := make(chan Batch, QueueSize)
chExtractorErrors := make(chan ExtractorError, QueueSize)
chChunksRaw := make(chan Chunk, QueueSize)
@@ -45,7 +45,12 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
var wgTransformers sync.WaitGroup
var wgLoaders sync.WaitGroup
go jobErrorHandler(ctx, chJobErrors)
go func() {
if err := jobErrorHandler(ctx, chJobErrors); err != nil {
cancel()
}
}()
go extractorErrorHandler(ctx, chExtractorErrors, chBatches, chJobErrors, &wgActiveBatches)
go loaderErrorHandler(ctx, chLoadersErrors, chChunksTransformed, chJobErrors, &wgActiveChunks)
@@ -80,7 +85,7 @@ func processMigrationJob(sourceDb *sql.DB, targetDb *pgxpool.Pool, job Migration
for range NumLoaders {
wgLoaders.Go(func() {
loadRowsPostgres(ctx, targetDb, job, targetColTypes, chChunksTransformed, chLoadersErrors, &wgActiveChunks)
loadRowsPostgres(ctx, targetDb, job, targetColTypes, chChunksTransformed, chLoadersErrors, chJobErrors, &wgActiveChunks)
})
}