74 lines
1.4 KiB
Go
74 lines
1.4 KiB
Go
package transformers
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"time"
|
|
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
|
|
)
|
|
|
|
const processBatchCtxCheck = 4096
|
|
|
|
func ProcessBatchWithRetries(
|
|
ctx context.Context,
|
|
batch *models.Batch,
|
|
transformationPlan []etl.ColumnTransformPlan,
|
|
retryConfig config.RetryConfig,
|
|
) error {
|
|
for i, rowValues := range batch.Rows {
|
|
if i%processBatchCtxCheck == 0 {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
for _, task := range transformationPlan {
|
|
val := rowValues[task.Index]
|
|
if val == nil {
|
|
continue
|
|
}
|
|
|
|
var lastErr error
|
|
success := false
|
|
|
|
for attempt := 0; attempt < retryConfig.Attempts; attempt++ {
|
|
transformed, err := task.Fn(val)
|
|
if err == nil {
|
|
rowValues[task.Index] = transformed
|
|
success = true
|
|
break
|
|
}
|
|
|
|
lastErr = err
|
|
if jobError, ok := errors.AsType[*custom_errors.JobError](err); ok {
|
|
if jobError.ShouldCancelJob {
|
|
return jobError
|
|
}
|
|
}
|
|
|
|
if attempt == retryConfig.Attempts-1 {
|
|
break
|
|
}
|
|
|
|
delay := custom_errors.ComputeBackoffDelay(
|
|
attempt,
|
|
retryConfig.BaseDelayMs,
|
|
retryConfig.MaxDelayMs,
|
|
retryConfig.MaxJitterMs,
|
|
)
|
|
time.Sleep(delay)
|
|
}
|
|
|
|
if !success {
|
|
return lastErr
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|