refactor: adjust configuration parameters for extractors and loaders; enhance logging messages for clarity

2026-05-09 01:48:36 -05:00
parent 5a8bce7701
commit a8be31c18b
4 changed files with 8 additions and 7 deletions
--- a/config.yaml
+++ b/config.yaml
@@ -70,14 +70,15 @@ jobs:
        - source: DATA
          target: FILE_URL
          mode: REFERENCE_ONLY
-    batches_per_partition: 10000
+    batches_per_partition: 1000
    max_extractors: 8
-    extractor_queue_size: 32
    extractor_batch_size: 1
+    extractor_queue_size: 64
    max_transformers: 16
-    transformer_batch_size: 20000
+    transformer_batch_size: 500
    transformer_queue_size: 8
    max_loaders: 4
+    loader_batch_size: 500
    retry:
      attempts: 5
      base_delay_ms: 1000