refactor: enhance config handling and unify main entrypoint
Unify configuration parsing logic across entrypoints and clean up legacy script-side config paths.
This commit is contained in:
49
internal/app/etl/loaders/process-with-retries.go
Normal file
49
internal/app/etl/loaders/process-with-retries.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package loaders
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
|
||||
)
|
||||
|
||||
func (gl *GenericLoader) ProcessBatchWithRetries(
|
||||
ctx context.Context,
|
||||
tableInfo config.TargetTableInfo,
|
||||
colNames []string,
|
||||
retryConfig config.RetryConfig,
|
||||
batch models.Batch,
|
||||
) (int64, error) {
|
||||
for {
|
||||
rowsLoaded, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch)
|
||||
if err == nil {
|
||||
return rowsLoaded, nil
|
||||
}
|
||||
|
||||
if btError, ok := errors.AsType[*custom_errors.LoaderError](err); ok {
|
||||
batch.RetryCounter++
|
||||
|
||||
if batch.RetryCounter >= retryConfig.Attempts {
|
||||
return rowsLoaded, &custom_errors.JobError{
|
||||
Msg: fmt.Sprintf("Batch %v reached max retries (%d)", batch.Id, batch.RetryCounter),
|
||||
Prev: btError,
|
||||
}
|
||||
}
|
||||
|
||||
delay := custom_errors.ComputeBackoffDelay(
|
||||
batch.RetryCounter,
|
||||
retryConfig.BaseDelayMs,
|
||||
retryConfig.MaxDelayMs,
|
||||
retryConfig.MaxJitterMs,
|
||||
)
|
||||
time.Sleep(delay)
|
||||
continue
|
||||
}
|
||||
|
||||
return rowsLoaded, err
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user