feat: add exponential backoff retry strategy

Introduce a retry policy with exponential backoff and jitter for extractor and loader errors, with configurable max attempts and delay caps.

internal/app/custom_errors/backoff.go

@@ -0,0 +1,61 @@
+package custom_errors
+
+import (
+	"context"
+	"math/rand"
+	"time"
+)
+
+func computeBackoffDelay(retryCounter int, baseDelayMs int, maxDelayMs int, maxJitterMs int) time.Duration {
+	if retryCounter < 0 {
+		retryCounter = 0
+	}
+
+	delay := max(time.Duration(baseDelayMs)*time.Millisecond, 0)
+
+	maxDelay := time.Duration(maxDelayMs) * time.Millisecond
+	for i := 0; i < retryCounter; i++ {
+		if maxDelayMs > 0 && delay >= maxDelay {
+			delay = maxDelay
+			break
+		}
+		if delay == 0 {
+			break
+		}
+		delay *= 2
+	}
+
+	if maxDelayMs > 0 && delay > maxDelay {
+		delay = maxDelay
+	}
+
+	if maxJitterMs > 0 {
+		jitter := time.Duration(rand.Intn(maxJitterMs+1)) * time.Millisecond
+		delay += jitter
+	}
+
+	if delay < 0 {
+		delay = 0
+	}
+
+	return delay
+}
+
+func requeueWithBackoff(ctx context.Context, delay time.Duration, enqueue func()) {
+	if delay <= 0 {
+		enqueue()
+		return
+	}
+
+	go func() {
+		timer := time.NewTimer(delay)
+		defer timer.Stop()
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			enqueue()
+		}
+	}()
+}

internal/app/custom_errors/extractor.error.go

@@ -5,12 +5,13 @@ import (
 	"fmt"
 	"sync"
 
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 	"github.com/google/uuid"
 )
 
 type ExtractorError struct {
-	Batch     models.Partition
+	Partition models.Partition
 	LastId    int64
 	HasLastId bool
 	Msg       string
@@ -22,11 +23,11 @@ func (e *ExtractorError) Error() string {
 
 func ExtractorErrorHandler(
 	ctx context.Context,
-	maxRetryAttempts int,
+	retryConfig config.RetryConfig,
 	chErrorsIn <-chan ExtractorError,
-	chBatchesOut chan<- models.Partition,
+	chPartitionsOut chan<- models.Partition,
 	chJobErrorsOut chan<- JobError,
-	wgActiveBatches *sync.WaitGroup,
+	wgActivePartitions *sync.WaitGroup,
 ) {
 	for {
 		if ctx.Err() != nil {
@@ -42,10 +43,11 @@ func ExtractorErrorHandler(
 				return
 			}
 
-			if err.Batch.RetryCounter >= maxRetryAttempts {
+			if err.Partition.RetryCounter >= retryConfig.Attempts {
+				wgActivePartitions.Done()
 				jobError := JobError{
 					ShouldCancelJob: false,
-					Msg:             fmt.Sprintf("batch %v reached max retries (%d)", err.Batch.Id, maxRetryAttempts),
+					Msg:             fmt.Sprintf("Partition %v reached max retries (%d)", err.Partition.Id, retryConfig.Attempts),
 					Prev:            &err,
 				}
 
@@ -55,25 +57,45 @@ func ExtractorErrorHandler(
 					return
 				}
 
-				wgActiveBatches.Done()
 				continue
+			} else {
+				jobError := JobError{
+					ShouldCancelJob: false,
+					Msg:             fmt.Sprintf("Temporal error in partition %v (retries: %d)", err.Partition.Id, err.Partition.RetryCounter),
+					Prev:            &err,
+				}
+
+				select {
+				case chJobErrorsOut <- jobError:
+				case <-ctx.Done():
+					return
+				}
 			}
 
-			newBatch := err.Batch
-			newBatch.RetryCounter++
+			newPartition := err.Partition
+			newPartition.RetryCounter++
+
+			delay := computeBackoffDelay(
+				newPartition.RetryCounter,
+				retryConfig.BaseDelayMs,
+				retryConfig.MaxDelayMs,
+				retryConfig.MaxJitterMs,
+			)
 
 			if err.HasLastId {
-				newBatch.ParentId = err.Batch.Id
-				newBatch.Id = uuid.New()
-				newBatch.LowerLimit = err.LastId
-				newBatch.IsLowerLimitInclusive = false
+				newPartition.ParentId = err.Partition.Id
+				newPartition.Id = uuid.New()
+				newPartition.LowerLimit = err.LastId
+				newPartition.IsLowerLimitInclusive = false
 			}
 
-			select {
-			case chBatchesOut <- newBatch:
-			case <-ctx.Done():
-				return
-			}
+			requeueWithBackoff(ctx, delay, func() {
+				select {
+				case chPartitionsOut <- newPartition:
+				case <-ctx.Done():
+					return
+				}
+			})
 		}
 	}
 }

internal/app/custom_errors/job.error.go

@@ -37,11 +37,11 @@ func JobErrorHandler(ctx context.Context, chErrorsIn <-chan JobError) error {
 			}
 
 			if err.ShouldCancelJob {
-				log.Error(err.Msg, " - ", err.Prev)
+				log.Errorf("(Fatal job error) - %v - %v", err.Msg, err.Prev)
 				return &err
 			}
 
-			log.Error(err.Msg, " - ", err.Prev)
+			log.Errorf("%v - %v", err.Msg, err.Prev)
 		}
 	}
 }

internal/app/custom_errors/loader.error.go

@@ -5,12 +5,13 @@ import (
 	"fmt"
 	"sync"
 
+	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
 	"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
 )
 
 type LoaderError struct {
-	models.Batch
-	Msg string
+	Batch models.Batch
+	Msg   string
 }
 
 func (e *LoaderError) Error() string {
@@ -19,11 +20,11 @@ func (e *LoaderError) Error() string {
 
 func LoaderErrorHandler(
 	ctx context.Context,
-	maxRetryAttempts int,
+	retryConfig config.RetryConfig,
 	chErrorsIn <-chan LoaderError,
-	chChunksOut chan<- models.Batch,
+	chBatchesOut chan<- models.Batch,
 	chJobErrorsOut chan<- JobError,
-	wgActiveChunks *sync.WaitGroup,
+	wgActiveBatches *sync.WaitGroup,
 ) {
 	for {
 		if ctx.Err() != nil {
@@ -39,10 +40,11 @@ func LoaderErrorHandler(
 				return
 			}
 
-			if err.RetryCounter >= maxRetryAttempts {
+			if err.Batch.RetryCounter >= retryConfig.Attempts {
+				wgActiveBatches.Done()
 				jobError := JobError{
 					ShouldCancelJob: false,
-					Msg:             fmt.Sprintf("chunk %v reached max retries (%d)", err.Id, maxRetryAttempts),
+					Msg:             fmt.Sprintf("Batch %v reached max retries (%d)", err.Batch.Id, retryConfig.Attempts),
 					Prev:            &err,
 				}
 
@@ -52,17 +54,36 @@ func LoaderErrorHandler(
 					return
 				}
 
-				wgActiveChunks.Done()
 				continue
+			} else {
+				jobError := JobError{
+					ShouldCancelJob: false,
+					Msg:             fmt.Sprintf("Temporal error in batch %v (retries: %d)", err.Batch.Id, err.Batch.RetryCounter),
+					Prev:            &err,
+				}
+
+				select {
+				case chJobErrorsOut <- jobError:
+				case <-ctx.Done():
+					return
+				}
 			}
 
-			err.RetryCounter++
+			err.Batch.RetryCounter++
+			delay := computeBackoffDelay(
+				err.Batch.RetryCounter,
+				retryConfig.BaseDelayMs,
+				retryConfig.MaxDelayMs,
+				retryConfig.MaxJitterMs,
+			)
 
-			select {
-			case chChunksOut <- err.Batch:
-			case <-ctx.Done():
-				return
-			}
+			requeueWithBackoff(ctx, delay, func() {
+				select {
+				case chBatchesOut <- err.Batch:
+				case <-ctx.Done():
+					return
+				}
+			})
 		}
 	}
 }