fix: enhance error handling in extractor and loader processes; ensure proper job error propagation and logging
This commit is contained in:
@@ -40,6 +40,7 @@ func LoaderErrorHandler(
|
||||
}
|
||||
|
||||
if err.RetryCounter >= maxRetryAttempts {
|
||||
wgActiveBatches.Done()
|
||||
jobError := JobError{
|
||||
ShouldCancelJob: false,
|
||||
Msg: fmt.Sprintf("Batch %v reached max retries (%d)", err.Id, maxRetryAttempts),
|
||||
@@ -52,8 +53,19 @@ func LoaderErrorHandler(
|
||||
return
|
||||
}
|
||||
|
||||
wgActiveBatches.Done()
|
||||
continue
|
||||
} else {
|
||||
jobError := JobError{
|
||||
ShouldCancelJob: false,
|
||||
Msg: fmt.Sprintf("Temporal error in batch %v (retries: %d)", err.Batch.Id, err.Batch.RetryCounter),
|
||||
Prev: &err,
|
||||
}
|
||||
|
||||
select {
|
||||
case chJobErrorsOut <- jobError:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err.RetryCounter++
|
||||
|
||||
Reference in New Issue
Block a user