refactor: update migration job configuration parameters; rename queue and batch size fields for clarity

2026-05-05 23:04:27 -05:00
parent 6414943cf3
commit 7cb959a103
3 changed files with 34 additions and 26 deletions
--- a/config.yaml
+++ b/config.yaml
@@ -3,15 +3,19 @@ source_db_type: sqlserver
 target_db_type: postgres

 defaults:
-  max_extractors: 2
-  max_loaders: 4
-  queue_size: 8
-  batch_size: 25000
  batches_per_partition: 8
+  max_extractors: 2
+  extractor_batch_size: 25000
+  extractor_queue_size: 8
+  max_transformers: 2
+  transformer_batch_size: 25000
+  transformer_queue_size: 8
+  max_loaders: 4
+  loader_batch_size: 25000
  truncate_target: true
  truncate_method: TRUNCATE # TRUNCATE | DELETE
  max_partition_errrors: 5
-  max_chunk_errors: 5
+  max_extractor_batch_errors: 5
  retry:
    attempts: 3
    base_delay_ms: 500