feat: update job configuration for infraestructura_site_holder__attach; enhance storage handling and retry logic

cmd/go_migrate/main.go

@@ -24,7 +24,7 @@ func main() {
 		log.Fatalf("error leyendo configuracion: %v", err)
 	}
 
-	log.Debugf("Config: %+v", migrationConfig)
+	// log.Debugf("Config: %+v", migrationConfig)
 
 	startTime := time.Now()
 

config.yaml

@@ -50,16 +50,25 @@ jobs:
       post_sql: 
         - "SELECT 1"
 
-  - name: red_terminal__attach
+  - name: infraestructura_site_holder__attach
     source:
-      schema: Red
-      table: TERMINAL__ATTACH
+      schema: Infraestructura
+      table: SITE_HOLDER__ATTACH
       primary_key: GDB_ARCHIVE_OID
     target:
-      schema: Red
-      table: TERMINAL__ATTACH
+      schema: Infraestructura
+      table: SITE_HOLDER__ATTACH
     to_storage:
       columns:
         - source: DATA
           target: FILE_URL
-          mode: REFERENCE_ONLY # REFERENCE_ONLY | DUPLICATE_WITH_REF
+          mode: REFERENCE_ONLY
+    max_extractors: 8
+    max_loaders: 4
+    queue_size: 32
+    batch_size: 1
+    retry:
+      attempts: 5
+      base_delay_ms: 1000
+      max_delay_ms: 15000
+      max_jitter_ms: 500

go.mod

@@ -4,11 +4,10 @@ go 1.26
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
-	github.com/caarlos0/env/v11 v11.4.0
 	github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4
 	github.com/google/uuid v1.6.0
+	github.com/ilyakaznacheev/cleanenv v1.5.0
 	github.com/jackc/pgx/v5 v5.9.1
-	github.com/joho/godotenv v1.5.1
 	github.com/microsoft/go-mssqldb v1.9.8
 	github.com/sirupsen/logrus v1.9.4
 	github.com/twpayne/go-geom v1.6.1
@@ -19,15 +18,18 @@ require (
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
+	github.com/BurntSushi/toml v1.6.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
 	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
+	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
 )

go.sum

@@ -14,14 +14,15 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBAD
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
+github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
 github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
 github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
 github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoGAc=
-github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -37,6 +38,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4=
+github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -88,3 +91,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ=
+olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw=

internal/app/config/main.go

@@ -1,8 +1,7 @@
 package config
 
 import (
-	"github.com/caarlos0/env/v11"
-	"github.com/joho/godotenv"
+	"github.com/ilyakaznacheev/cleanenv"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -10,28 +9,30 @@ type AzureStorageConfig struct {
 	AccountName string `env:"AZ_ACCOUNT_NAME"`
 	Container   string `env:"AZ_CONTAINER"`
 	AccountKey  string `env:"AZ_ACCOUNT_KEY"`
-	UseHTTPS    bool   `env:"AZ_USE_HTTPS" envDefault:"true"`
+	UseHTTPS    bool   `env:"AZ_USE_HTTPS" env-default:"true"`
 	ServiceURL  string `env:"AZ_SERVICE_URL"`
 	Prefix      string `env:"AZ_PREFIX"`
 	Enabled     bool   `env:"AZ_STORAGE_ENABLED"`
 }
 
 type appConfig struct {
-	SourceDbUrl  string `env:"SOURCE_DB_URL,required"`
-	TargetDbUrl  string `env:"TARGET_DB_URL,required"`
-	LogLevel     string `env:"LOG_LEVEL" envDefault:"INFO"`
+	SourceDbUrl  string `env:"SOURCE_DB_URL" env-required:"true"`
+	TargetDbUrl  string `env:"TARGET_DB_URL" env-required:"true"`
+	LogLevel     string `env:"LOG_LEVEL" env-default:"INFO"`
 	AzureStorage AzureStorageConfig
 }
 
 func getAppConfig() appConfig {
-	if err := godotenv.Load(); err != nil {
+	var cfg appConfig
+
+	err := cleanenv.ReadConfig(".env", &cfg)
+	if err != nil {
 		log.Warn("Could not load .env file")
 	}
 
-	cfg := appConfig{}
-
-	if err := env.Parse(&cfg); err != nil {
-		log.Fatalf("Error al cargar variables de entorno: %v", err)
+	err = cleanenv.ReadEnv(&cfg)
+	if err != nil {
+		log.Fatalf("Error al cargar variables: %v", err)
 	}
 
 	return cfg

internal/app/etl/transformers/mssql.go

@@ -119,11 +119,13 @@ func computeStorageTransformationPlan(
 						schema, table, sourceColName, v)
 					return v, nil
 				}
-				blobPath := fmt.Sprintf("%s/%s/%s/%s.bin", schema, table, sourceColName, uuid.New().String())
+				start := time.Now()
+				blobPath := fmt.Sprintf("%s/%s/%s", schema, table, uuid.New().String())
 				blobURL, err := azureClient.UploadAndGetURL(ctx, blobPath, b)
 				if err != nil {
 					return nil, fmt.Errorf("uploading %s.%s.%s: %w", schema, table, sourceColName, err)
 				}
+				log.Debugf(`Succesfully uploaded "%s", (%vms)`, blobURL, time.Since(start).Milliseconds())
 				return blobURL, nil
 			},
 		})