refactor: remove unused error channels and enhance job configuration; add max failed batches load parameter

config.yaml

@@ -12,10 +12,11 @@ defaults:
   transformer_queue_size: 8
   max_loaders: 4
   loader_batch_size: 25000
+  max_failed_partitions: 5
+  max_failed_batches_transform: 5
+  max_failed_batches_load: 5
   truncate_target: true
   truncate_method: TRUNCATE # TRUNCATE | DELETE
-  max_partition_errrors: 5
-  max_extractor_batch_errors: 5
   retry:
     attempts: 3
     base_delay_ms: 500
@@ -34,11 +35,11 @@ jobs:
       table: MANZANA
       pre_sql: 
         - 'SELECT 1'
-    range:
-      min: 1000000
-      max: 2000000
-      is_min_inclusive: false
-      is_max_inclusive: true
+    # range:
+    #   min: 1000000
+    #   max: 2000000
+    #   is_min_inclusive: false
+    #   is_max_inclusive: true
 
   - name: red_puerto
     enabled: true
@@ -57,25 +58,29 @@ jobs:
       post_sql: 
         - "SELECT 1"
 
-  - name: infraestructura_site_holder__attach
-    source:
-      schema: Infraestructura
-      table: SITE_HOLDER__ATTACH
-      primary_key: GDB_ARCHIVE_OID
-    target:
-      schema: Infraestructura
-      table: SITE_HOLDER__ATTACH
-    to_storage:
-      columns:
-        - source: DATA
-          target: FILE_URL
-          mode: REFERENCE_ONLY
-    max_extractors: 8
-    max_loaders: 4
-    queue_size: 32
-    batch_size: 1
-    retry:
-      attempts: 5
-      base_delay_ms: 1000
-      max_delay_ms: 15000
-      max_jitter_ms: 500
+  # - name: infraestructura_site_holder__attach
+  #   source:
+  #     schema: Infraestructura
+  #     table: SITE_HOLDER__ATTACH
+  #     primary_key: GDB_ARCHIVE_OID
+  #   target:
+  #     schema: Infraestructura
+  #     table: SITE_HOLDER__ATTACH
+  #   to_storage:
+  #     columns:
+  #       - source: DATA
+  #         target: FILE_URL
+  #         mode: REFERENCE_ONLY
+  #   batches_per_partition: 10000
+  #   max_extractors: 8
+  #   extractor_queue_size: 32
+  #   extractor_batch_size: 1
+  #   max_transformers: 16
+  #   transformer_batch_size: 20000
+  #   transformer_queue_size: 8
+  #   max_loaders: 4
+  #   retry:
+  #     attempts: 5
+  #     base_delay_ms: 1000
+  #     max_delay_ms: 15000
+  #     max_jitter_ms: 500