refactor: enhance logging and batch processing in migration; adjust configuration parameters for improved performance

config.yaml

@@ -3,12 +3,12 @@ source_db_type: sqlserver
 target_db_type: postgres
 
 defaults:
-  batches_per_partition: 8
+  batches_per_partition: 4
   max_extractors: 2
-  extractor_batch_size: 25000
+  extractor_batch_size: 5000
   extractor_queue_size: 8
   max_transformers: 2
-  transformer_batch_size: 25000
+  transformer_batch_size: 12500
   transformer_queue_size: 8
   max_loaders: 4
   loader_batch_size: 25000
@@ -34,11 +34,11 @@ jobs:
       table: MANZANA
       pre_sql: 
         - 'SELECT 1'
-    # range:
-    #   min: 1000000
-    #   max: 2000000
-    #   is_min_inclusive: false
-    #   is_max_inclusive: true
+    range:
+      min: 1000000
+      max: 2000000
+      is_min_inclusive: false
+      is_max_inclusive: true
 
   - name: red_puerto
     enabled: true
@@ -57,29 +57,29 @@ jobs:
       post_sql: 
         - "SELECT 1"
 
-  # - name: infraestructura_site_holder__attach
-  #   source:
-  #     schema: Infraestructura
-  #     table: SITE_HOLDER__ATTACH
-  #     primary_key: GDB_ARCHIVE_OID
-  #   target:
-  #     schema: Infraestructura
-  #     table: SITE_HOLDER__ATTACH
-  #   to_storage:
-  #     columns:
-  #       - source: DATA
-  #         target: FILE_URL
-  #         mode: REFERENCE_ONLY
-  #   batches_per_partition: 10000
-  #   max_extractors: 8
-  #   extractor_queue_size: 32
-  #   extractor_batch_size: 1
-  #   max_transformers: 16
-  #   transformer_batch_size: 20000
-  #   transformer_queue_size: 8
-  #   max_loaders: 4
-  #   retry:
-  #     attempts: 5
-  #     base_delay_ms: 1000
-  #     max_delay_ms: 15000
-  #     max_jitter_ms: 500
+  - name: infraestructura_site_holder__attach
+    source:
+      schema: Infraestructura
+      table: SITE_HOLDER__ATTACH
+      primary_key: GDB_ARCHIVE_OID
+    target:
+      schema: Infraestructura
+      table: SITE_HOLDER__ATTACH
+    to_storage:
+      columns:
+        - source: DATA
+          target: FILE_URL
+          mode: REFERENCE_ONLY
+    batches_per_partition: 10000
+    max_extractors: 8
+    extractor_queue_size: 32
+    extractor_batch_size: 1
+    max_transformers: 16
+    transformer_batch_size: 20000
+    transformer_queue_size: 8
+    max_loaders: 4
+    retry:
+      attempts: 5
+      base_delay_ms: 1000
+      max_delay_ms: 15000
+      max_jitter_ms: 500