feat: update job configuration for infraestructura_site_holder__attach; enhance storage handling and retry logic

config.yaml

@@ -50,16 +50,25 @@ jobs:
       post_sql: 
         - "SELECT 1"
 
-  - name: red_terminal__attach
+  - name: infraestructura_site_holder__attach
     source:
-      schema: Red
+      schema: Infraestructura
-      table: TERMINAL__ATTACH
+      table: SITE_HOLDER__ATTACH
       primary_key: GDB_ARCHIVE_OID
     target:
-      schema: Red
+      schema: Infraestructura
-      table: TERMINAL__ATTACH
+      table: SITE_HOLDER__ATTACH
     to_storage:
       columns:
         - source: DATA
           target: FILE_URL
-          mode: REFERENCE_ONLY # REFERENCE_ONLY | DUPLICATE_WITH_REF
+          mode: REFERENCE_ONLY
+    max_extractors: 8
+    max_loaders: 4
+    queue_size: 32
+    batch_size: 1
+    retry:
+      attempts: 5
+      base_delay_ms: 1000
+      max_delay_ms: 15000
+      max_jitter_ms: 500

internal/app/etl/transformers/mssql.go

@@ -119,11 +119,13 @@ func computeStorageTransformationPlan(
 						schema, table, sourceColName, v)
 					return v, nil
 				}
-				blobPath := fmt.Sprintf("%s/%s/%s/%s.bin", schema, table, sourceColName, uuid.New().String())
+				start := time.Now()
+				blobPath := fmt.Sprintf("%s/%s/%s", schema, table, uuid.New().String())
 				blobURL, err := azureClient.UploadAndGetURL(ctx, blobPath, b)
 				if err != nil {
 					return nil, fmt.Errorf("uploading %s.%s.%s: %w", schema, table, sourceColName, err)
 				}
+				log.Debugf(`Succesfully uploaded "%s", (%vms)`, blobURL, time.Since(start).Milliseconds())
 				return blobURL, nil
 			},
 		})