refactor: remove unused error channels and enhance job configuration; add max failed batches load parameter

2026-05-08 22:28:31 -05:00
parent c4e233401b
commit 212d3663e2
4 changed files with 54 additions and 59 deletions
--- a/internal/app/etl/loaders/process-with-retries.go
+++ b/internal/app/etl/loaders/process-with-retries.go
@@ -18,6 +18,7 @@ func (gl *GenericLoader) ProcessBatchWithRetries(
 	retryConfig config.RetryConfig,
 	batch models.Batch,
 ) (int64, error) {
+	retries := 0
 	for {
 		rowsLoaded, err := gl.ProcessBatch(ctx, tableInfo, colNames, batch)
 		if err == nil {
@@ -25,7 +26,8 @@ func (gl *GenericLoader) ProcessBatchWithRetries(
 		}

 		if btError, ok := errors.AsType[*custom_errors.LoaderError](err); ok {
-			batch.RetryCounter++
+			retries++
+			batch.RetryCounter = retries

 			if batch.RetryCounter >= retryConfig.Attempts {
 				return rowsLoaded, &custom_errors.JobError{