feat: enhance migration job processing with detailed metrics and error handling

This commit is contained in:
2026-04-09 21:55:19 -05:00
parent 1db35c796c
commit 6345a0d694
5 changed files with 86 additions and 21 deletions

View File

@@ -36,7 +36,25 @@ func main() {
defer sourceDb.Close()
defer targetDb.Close()
processMigrationJobs(ctx, sourceDb, targetDb, migrationConfig.Jobs, migrationConfig.MaxParallelWorkers)
results := processMigrationJobs(ctx, sourceDb, targetDb, migrationConfig.Jobs, migrationConfig.MaxParallelWorkers)
log.Info("=== RESUMEN DE MIGRACIÓN ===")
var totalProcessed, totalErrors int64
for _, res := range results {
status := "OK"
if res.Error != nil {
status = "FAILED"
}
log.Infof("[%s] Status: %s | Read: %d | Loaded: %d | Errors: %d | Time: %v", res.JobName, status, res.RowsRead, res.RowsLoaded, res.RowsFailed, res.Duration)
totalProcessed += res.RowsLoaded
if res.Error != nil {
totalErrors++
}
}
log.Infof("Migración terminada. Tablas: %d, Errores: %d, Filas totales: %d", len(results), totalErrors, totalProcessed)
totalDuration := time.Since(startTime)
log.Infof("=== Migration completed successfully! ===")
@@ -49,10 +67,10 @@ func processMigrationJobs(
targetDb *pgxpool.Pool,
jobs []config.Job,
maxParallelWorkers int,
) {
) []JobResult {
if len(jobs) == 0 {
log.Info("No migration jobs configured")
return
return []JobResult{}
}
if maxParallelWorkers <= 0 {
@@ -65,6 +83,7 @@ func processMigrationJobs(
log.Infof("Starting migration with %d parallel worker(s)", maxParallelWorkers)
chJobResults := make(chan JobResult, len(jobs))
chJobs := make(chan config.Job, len(jobs))
var wgJobs sync.WaitGroup
@@ -72,7 +91,8 @@ func processMigrationJobs(
wgJobs.Go(func() {
for job := range chJobs {
log.Infof("[worker %d] >>> Processing job: %s.%s <<<", i, job.SourceTable.Schema, job.SourceTable.Table)
processMigrationJob(ctx, sourceDb, targetDb, job)
res := processMigrationJob(ctx, sourceDb, targetDb, job)
chJobResults <- res
}
})
}
@@ -80,8 +100,17 @@ func processMigrationJobs(
for _, job := range jobs {
chJobs <- job
}
close(chJobs)
wgJobs.Wait()
go func() {
wgJobs.Wait()
close(chJobResults)
}()
var finalResults []JobResult
for res := range chJobResults {
finalResults = append(finalResults, res)
}
return finalResults
}