feat: integrate azure storage handling in migration pipeline

Stream transformer output to an Azure blob container when a job declares ToStorage, with per-job prefix and credential configuration.

internal/app/config/main.go

@@ -1,41 +1,40 @@
 package config
 
 import (
-	"os"
-
+	"github.com/caarlos0/env/v11"
 	"github.com/joho/godotenv"
 	log "github.com/sirupsen/logrus"
 )
 
-type appConfig struct {
-	SourceDbUrl string
-	TargetDbUrl string
+type AzureStorageConfig struct {
+	AccountName string `env:"AZ_ACCOUNT_NAME"`
+	Container   string `env:"AZ_CONTAINER"`
+	AccountKey  string `env:"AZ_ACCOUNT_KEY"`
+	UseHTTPS    bool   `env:"AZ_USE_HTTPS" envDefault:"true"`
+	ServiceURL  string `env:"AZ_SERVICE_URL"`
+	Prefix      string `env:"AZ_PREFIX"`
+	Enabled     bool   `env:"AZ_STORAGE_ENABLED"`
 }
 
-func loadEnv() {
-	err := godotenv.Load()
-	if err != nil {
-		log.Warn("Warning: could not load .env file")
-	}
+type appConfig struct {
+	SourceDbUrl  string `env:"SOURCE_DB_URL,required"`
+	TargetDbUrl  string `env:"TARGET_DB_URL,required"`
+	LogLevel     string `env:"LOG_LEVEL" envDefault:"INFO"`
+	AzureStorage AzureStorageConfig
 }
 
 func getAppConfig() appConfig {
-	loadEnv()
-
-	sourceDbUrl := os.Getenv("SOURCE_DB_URL")
-	if sourceDbUrl == "" {
-		log.Fatal("SOURCE_DB_URL environment variable not set")
+	if err := godotenv.Load(); err != nil {
+		log.Warn("Could not load .env file")
 	}
 
-	targetDbUrl := os.Getenv("TARGET_DB_URL")
-	if targetDbUrl == "" {
-		log.Fatal("TARGET_DB_URL environment variable not set")
+	cfg := appConfig{}
+
+	if err := env.Parse(&cfg); err != nil {
+		log.Fatalf("Error al cargar variables de entorno: %v", err)
 	}
 
-	return appConfig{
-		SourceDbUrl: sourceDbUrl,
-		TargetDbUrl: targetDbUrl,
-	}
+	return cfg
 }
 
 var App appConfig = getAppConfig()

internal/app/config/migration.go

@@ -14,6 +14,16 @@ type RetryConfig struct {
 	MaxJitterMs int `yaml:"max_jitter_ms"`
 }
 
+type ToStorageColumnConfig struct {
+	Source string `yaml:"source"`
+	Target string `yaml:"target"`
+	Mode   string `yaml:"mode"`
+}
+
+type ToStorageConfig struct {
+	Columns []ToStorageColumnConfig `yaml:"columns"`
+}
+
 type JobConfig struct {
 	MaxExtractors       int         `yaml:"max_extractors"`
 	MaxLoaders          int         `yaml:"max_loaders"`
@@ -26,6 +36,7 @@ type JobConfig struct {
 	MaxChunkErrors      int         `yaml:"max_chunk_errors"`
 	Retry               RetryConfig `yaml:"retry"`
 	RowsPerPartition    int64
+	ToStorage           ToStorageConfig `yaml:"to_storage"`
 }
 
 type TableInfo struct {
@@ -33,29 +44,31 @@ type TableInfo struct {
 	Table  string `yaml:"table"`
 }
 
-type TargetTableInfo struct {
-	TableInfo `yaml:",inline"`
-}
-
 type SourceTableInfo struct {
 	TableInfo  `yaml:",inline"`
 	PrimaryKey string `yaml:"primary_key"`
 }
 
+type TargetTableInfo struct {
+	TableInfo `yaml:",inline"`
+	PreSQL    []string `yaml:"pre_sql"`
+	PostSQL   []string `yaml:"post_sql"`
+}
+
+type RangeConfig struct {
+	Min            int64 `yaml:"min"`
+	Max            int64 `yaml:"max"`
+	IsMinInclusive bool  `yaml:"is_min_inclusive"`
+	IsMaxInclusive bool  `yaml:"is_max_inclusive"`
+}
+
 type Job struct {
 	Name        string          `yaml:"name"`
 	Enabled     bool            `yaml:"enabled"`
 	SourceTable SourceTableInfo `yaml:"source"`
 	TargetTable TargetTableInfo `yaml:"target"`
-	PreSQL      []string        `yaml:"pre_sql"`
-	PostSQL     []string        `yaml:"post_sql"`
 	JobConfig   `yaml:",inline"`
-	Range       struct {
-		Min            int64 `yaml:"min"`
-		Max            int64 `yaml:"max"`
-		IsMinInclusive bool  `yaml:"is_min_inclusive"`
-		IsMaxInclusive bool  `yaml:"is_max_inclusive"`
-	}
+	Range       RangeConfig `yaml:"range"`
 }
 
 type MigrationConfig struct {