diff --git a/config.yaml b/config.yaml index 6c3dd8c..63ac1c4 100644 --- a/config.yaml +++ b/config.yaml @@ -50,16 +50,25 @@ jobs: post_sql: - "SELECT 1" - - name: red_terminal__attach + - name: infraestructura_site_holder__attach source: - schema: Red - table: TERMINAL__ATTACH + schema: Infraestructura + table: SITE_HOLDER__ATTACH primary_key: GDB_ARCHIVE_OID target: - schema: Red - table: TERMINAL__ATTACH + schema: Infraestructura + table: SITE_HOLDER__ATTACH to_storage: columns: - source: DATA target: FILE_URL - mode: REFERENCE_ONLY # REFERENCE_ONLY | DUPLICATE_WITH_REF + mode: REFERENCE_ONLY + max_extractors: 8 + max_loaders: 4 + queue_size: 32 + batch_size: 1 + retry: + attempts: 5 + base_delay_ms: 1000 + max_delay_ms: 15000 + max_jitter_ms: 500 diff --git a/internal/app/etl/transformers/mssql.go b/internal/app/etl/transformers/mssql.go index 0500a6e..0c238e5 100644 --- a/internal/app/etl/transformers/mssql.go +++ b/internal/app/etl/transformers/mssql.go @@ -119,11 +119,13 @@ func computeStorageTransformationPlan( schema, table, sourceColName, v) return v, nil } - blobPath := fmt.Sprintf("%s/%s/%s/%s.bin", schema, table, sourceColName, uuid.New().String()) + start := time.Now() + blobPath := fmt.Sprintf("%s/%s/%s", schema, table, uuid.New().String()) blobURL, err := azureClient.UploadAndGetURL(ctx, blobPath, b) if err != nil { return nil, fmt.Errorf("uploading %s.%s.%s: %w", schema, table, sourceColName, err) } + log.Debugf(`Succesfully uploaded "%s", (%vms)`, blobURL, time.Since(start).Milliseconds()) return blobURL, nil }, })