feat: add exponential backoff retry strategy

Introduce a retry policy with exponential backoff and jitter for extractor and loader errors, with configurable max attempts and delay caps.
2026-04-13 14:00:00 -05:00
parent 7f3d2b8cc4
commit 74abf12dcf
21 changed files with 847 additions and 684 deletions
--- a/config.yaml
+++ b/config.yaml
@@ -6,12 +6,15 @@ defaults:
  max_extractors: 2
  max_loaders: 4
  queue_size: 8
-  chunk_size: 25000
-  chunks_per_batch: 8
+  batch_size: 25000
+  batches_per_partition: 8
  truncate_target: true
  truncate_method: TRUNCATE # TRUNCATE | DELETE
  retry:
    attempts: 3
+    base_delay_ms: 500
+    max_delay_ms: 10000
+    max_jitter_ms: 500

 jobs:
  - name: demo_users