Introduce a retry policy with exponential backoff and jitter for extractor and loader errors, with configurable max attempts and delay caps.
90 lines
1.7 KiB
Go
90 lines
1.7 KiB
Go
package custom_errors
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
|
|
)
|
|
|
|
type LoaderError struct {
|
|
Batch models.Batch
|
|
Msg string
|
|
}
|
|
|
|
func (e *LoaderError) Error() string {
|
|
return e.Msg
|
|
}
|
|
|
|
func LoaderErrorHandler(
|
|
ctx context.Context,
|
|
retryConfig config.RetryConfig,
|
|
chErrorsIn <-chan LoaderError,
|
|
chBatchesOut chan<- models.Batch,
|
|
chJobErrorsOut chan<- JobError,
|
|
wgActiveBatches *sync.WaitGroup,
|
|
) {
|
|
for {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
|
|
case err, ok := <-chErrorsIn:
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
if err.Batch.RetryCounter >= retryConfig.Attempts {
|
|
wgActiveBatches.Done()
|
|
jobError := JobError{
|
|
ShouldCancelJob: false,
|
|
Msg: fmt.Sprintf("Batch %v reached max retries (%d)", err.Batch.Id, retryConfig.Attempts),
|
|
Prev: &err,
|
|
}
|
|
|
|
select {
|
|
case chJobErrorsOut <- jobError:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
|
|
continue
|
|
} else {
|
|
jobError := JobError{
|
|
ShouldCancelJob: false,
|
|
Msg: fmt.Sprintf("Temporal error in batch %v (retries: %d)", err.Batch.Id, err.Batch.RetryCounter),
|
|
Prev: &err,
|
|
}
|
|
|
|
select {
|
|
case chJobErrorsOut <- jobError:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
|
|
err.Batch.RetryCounter++
|
|
delay := computeBackoffDelay(
|
|
err.Batch.RetryCounter,
|
|
retryConfig.BaseDelayMs,
|
|
retryConfig.MaxDelayMs,
|
|
retryConfig.MaxJitterMs,
|
|
)
|
|
|
|
requeueWithBackoff(ctx, delay, func() {
|
|
select {
|
|
case chBatchesOut <- err.Batch:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|