fix: enhance error handling in extractor and loader processes; ensure proper job error propagation and logging

This commit is contained in:
2026-04-12 20:11:19 -05:00
parent 01780b4b02
commit 5633dc98d0
5 changed files with 46 additions and 14 deletions

View File

@@ -43,6 +43,7 @@ func ExtractorErrorHandler(
}
if err.Partition.RetryCounter >= maxRetryAttempts {
wgActivePartitions.Done()
jobError := JobError{
ShouldCancelJob: false,
Msg: fmt.Sprintf("Partition %v reached max retries (%d)", err.Partition.Id, maxRetryAttempts),
@@ -55,8 +56,19 @@ func ExtractorErrorHandler(
return
}
wgActivePartitions.Done()
continue
} else {
jobError := JobError{
ShouldCancelJob: false,
Msg: fmt.Sprintf("Temporal error in partition %v (retries: %d)", err.Partition.Id, err.Partition.RetryCounter),
Prev: &err,
}
select {
case chJobErrorsOut <- jobError:
case <-ctx.Done():
return
}
}
newPartition := err.Partition