fix: enhance error handling in extractor and loader processes; ensure proper job error propagation and logging
This commit is contained in:
@@ -43,6 +43,7 @@ func ExtractorErrorHandler(
|
||||
}
|
||||
|
||||
if err.Partition.RetryCounter >= maxRetryAttempts {
|
||||
wgActivePartitions.Done()
|
||||
jobError := JobError{
|
||||
ShouldCancelJob: false,
|
||||
Msg: fmt.Sprintf("Partition %v reached max retries (%d)", err.Partition.Id, maxRetryAttempts),
|
||||
@@ -55,8 +56,19 @@ func ExtractorErrorHandler(
|
||||
return
|
||||
}
|
||||
|
||||
wgActivePartitions.Done()
|
||||
continue
|
||||
} else {
|
||||
jobError := JobError{
|
||||
ShouldCancelJob: false,
|
||||
Msg: fmt.Sprintf("Temporal error in partition %v (retries: %d)", err.Partition.Id, err.Partition.RetryCounter),
|
||||
Prev: &err,
|
||||
}
|
||||
|
||||
select {
|
||||
case chJobErrorsOut <- jobError:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
newPartition := err.Partition
|
||||
|
||||
Reference in New Issue
Block a user