feat: enhance migration job processing with detailed metrics and error handling
This commit is contained in:
@@ -36,7 +36,25 @@ func main() {
|
||||
defer sourceDb.Close()
|
||||
defer targetDb.Close()
|
||||
|
||||
processMigrationJobs(ctx, sourceDb, targetDb, migrationConfig.Jobs, migrationConfig.MaxParallelWorkers)
|
||||
results := processMigrationJobs(ctx, sourceDb, targetDb, migrationConfig.Jobs, migrationConfig.MaxParallelWorkers)
|
||||
|
||||
log.Info("=== RESUMEN DE MIGRACIÓN ===")
|
||||
var totalProcessed, totalErrors int64
|
||||
|
||||
for _, res := range results {
|
||||
status := "OK"
|
||||
if res.Error != nil {
|
||||
status = "FAILED"
|
||||
}
|
||||
log.Infof("[%s] Status: %s | Read: %d | Loaded: %d | Errors: %d | Time: %v", res.JobName, status, res.RowsRead, res.RowsLoaded, res.RowsFailed, res.Duration)
|
||||
|
||||
totalProcessed += res.RowsLoaded
|
||||
if res.Error != nil {
|
||||
totalErrors++
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("Migración terminada. Tablas: %d, Errores: %d, Filas totales: %d", len(results), totalErrors, totalProcessed)
|
||||
|
||||
totalDuration := time.Since(startTime)
|
||||
log.Infof("=== Migration completed successfully! ===")
|
||||
@@ -49,10 +67,10 @@ func processMigrationJobs(
|
||||
targetDb *pgxpool.Pool,
|
||||
jobs []config.Job,
|
||||
maxParallelWorkers int,
|
||||
) {
|
||||
) []JobResult {
|
||||
if len(jobs) == 0 {
|
||||
log.Info("No migration jobs configured")
|
||||
return
|
||||
return []JobResult{}
|
||||
}
|
||||
|
||||
if maxParallelWorkers <= 0 {
|
||||
@@ -65,6 +83,7 @@ func processMigrationJobs(
|
||||
|
||||
log.Infof("Starting migration with %d parallel worker(s)", maxParallelWorkers)
|
||||
|
||||
chJobResults := make(chan JobResult, len(jobs))
|
||||
chJobs := make(chan config.Job, len(jobs))
|
||||
var wgJobs sync.WaitGroup
|
||||
|
||||
@@ -72,7 +91,8 @@ func processMigrationJobs(
|
||||
wgJobs.Go(func() {
|
||||
for job := range chJobs {
|
||||
log.Infof("[worker %d] >>> Processing job: %s.%s <<<", i, job.SourceTable.Schema, job.SourceTable.Table)
|
||||
processMigrationJob(ctx, sourceDb, targetDb, job)
|
||||
res := processMigrationJob(ctx, sourceDb, targetDb, job)
|
||||
chJobResults <- res
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -80,8 +100,17 @@ func processMigrationJobs(
|
||||
for _, job := range jobs {
|
||||
chJobs <- job
|
||||
}
|
||||
|
||||
close(chJobs)
|
||||
|
||||
wgJobs.Wait()
|
||||
go func() {
|
||||
wgJobs.Wait()
|
||||
close(chJobResults)
|
||||
}()
|
||||
|
||||
var finalResults []JobResult
|
||||
for res := range chJobResults {
|
||||
finalResults = append(finalResults, res)
|
||||
}
|
||||
|
||||
return finalResults
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user