kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: enhance error messages for max retries in partition and batch processing

Browse Source
This commit is contained in:
Kylesoda
2026-05-08 23:36:59 -05:00
parent b5fd6d0534
commit d124da8b20
2 changed files with 4 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/app/etl/extractors/process-with-retries.go
View File

@@ -50,7 +50,7 @@ func (ex *GenericExtractor) ProcessPartitionWithRetries(
if currentParitition.RetryCounter >= retryConfig.Attempts { if currentParitition.RetryCounter >= retryConfig.Attempts {
return totalRowsRead, &custom_errors.JobError{ return totalRowsRead, &custom_errors.JobError{
Msg: fmt.Sprintf("Partition %v reached max retries", exError.Partition.Id), Msg: fmt.Sprintf("Partition %v reached max retries (%d)", currentParitition.Id, currentParitition.RetryCounter),
Prev: err, Prev: err,
} }
} }

7
internal/app/etl/loaders/process-with-retries.go
View File

@@ -18,7 +18,6 @@ func (gl *GenericLoader) ProcessBatchWithRetries(
retryConfig config.RetryConfig, retryConfig config.RetryConfig,
batch models.Batch, batch models.Batch,
) (int64, error) { ) (int64, error) {
retries := 0
for { for {
rowsLoaded, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch) rowsLoaded, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch)
if err == nil { if err == nil {
@@ -26,13 +25,11 @@ func (gl *GenericLoader) ProcessBatchWithRetries(
} }
if btError, ok := errors.AsType[*custom_errors.LoaderError](err); ok { if btError, ok := errors.AsType[*custom_errors.LoaderError](err); ok {
retries++ batch.RetryCounter++
batch.RetryCounter = retries
if batch.RetryCounter >= retryConfig.Attempts { if batch.RetryCounter >= retryConfig.Attempts {
return rowsLoaded, &custom_errors.JobError{ return rowsLoaded, &custom_errors.JobError{
ShouldCancelJob: false, Msg: fmt.Sprintf("Batch %v reached max retries (%d)", batch.Id, batch.RetryCounter),
Msg: fmt.Sprintf("Temporal error in batch %v (retries: %d)", btError.Batch.Id, btError.Batch.RetryCounter),
Prev: btError, Prev: btError,
} }
} }