refactor: adjust configuration parameters for extractors and transformers; optimize batch processing settings

2026-05-09 02:07:09 -05:00
parent 6fa9b21b1c
commit 0c59d06af6
2 changed files with 6 additions and 6 deletions
--- a/config.yaml
+++ b/config.yaml
@@ -70,11 +70,11 @@ jobs:
        - source: DATA
          target: FILE_URL
          mode: REFERENCE_ONLY
-    batches_per_partition: 1000
-    max_extractors: 8
+    batches_per_partition: 20
+    max_extractors: 32
    extractor_batch_size: 1
-    extractor_queue_size: 64
-    max_transformers: 16
+    extractor_queue_size: 100
+    max_transformers: 48
    transformer_batch_size: 500
    transformer_queue_size: 8
    max_loaders: 4