diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..31a74d3 --- /dev/null +++ b/config.yaml @@ -0,0 +1,46 @@ +max_parallel_workers: 2 + +defaults: + max_extractors: 4 + max_loaders: 8 + queue_size: 8 + chunk_size: 50000 + chunks_per_batch: 10 + truncate_target: true + truncate_method: TRUNCATE # TRUNCATE | DELETE + retry: + attempts: 3 + +jobs: + - name: cartografia_manzana + enabled: true + source: + schema: Cartografia + table: MANZANA + primary_key: GDB_ARCHIVE_OID + target: + schema: Cartografia + table: MANZANA + max_extractors: 2 # overrides default config + max_loaders: 4 # overrides default config + queue_size: 4 # overrides default config + chunk_size: 25000 # overrides default config + chunks_per_batch: 8 # overrides default config + truncate_target: false # overrides default config + truncate_method: DELETE # overrides default config + retry: + attempts: 5 # overrides default config + pre_sql: + - "SELECT 1" + post_sql: + - "SELECT 2" + + - name: red_puerto + enabled: true + source: + schema: Red + table: PUERTO + primary_key: ID_PUERTO + target: + schema: Red + table: PUERTO diff --git a/go.mod b/go.mod index e470a39..8d3a13e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.7 require ( github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4 + github.com/goccy/go-yaml v1.19.2 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.9.1 github.com/joho/godotenv v1.5.1 diff --git a/go.sum b/go.sum index 2bf97bb..33febce 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4 h1:4vH4+3zfwZTqoJEFw7DsTaH1V8jgVwnyeDvNi2TxzAc= github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4/go.mod h1:jlB0I5BIfcJBGdV6rRGPthSBfeY86RGkSAwcsldbHJc= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= diff --git a/scripts/config-parser/main.go b/scripts/config-parser/main.go new file mode 100644 index 0000000..a90d1ae --- /dev/null +++ b/scripts/config-parser/main.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "log" + "os" + + "github.com/goccy/go-yaml" +) + +// Estructuras para mapear el YAML +type RetryConfig struct { + Attempts int `yaml:"attempts"` +} + +type DBInfo struct { + Schema string `yaml:"schema"` + Table string `yaml:"table"` + PrimaryKey string `yaml:"primary_key,omitempty"` // omitempty si no siempre existe +} + +// JobSettings contiene los campos que se comparten entre 'defaults' y cada 'job' +type JobSettings struct { + MaxExtractors *int `yaml:"max_extractors"` + MaxLoaders *int `yaml:"max_loaders"` + QueueSize *int `yaml:"queue_size"` + ChunkSize *int `yaml:"chunk_size"` + ChunksPerBatch *int `yaml:"chunks_per_batch"` + TruncateTarget *bool `yaml:"truncate_target"` + TruncateMethod *string `yaml:"truncate_method"` + Retry *RetryConfig `yaml:"retry"` +} + +type Job struct { + Name string `yaml:"name"` + Enabled bool `yaml:"enabled"` + Source DBInfo `yaml:"source"` + Target DBInfo `yaml:"target"` + PreSQL []string `yaml:"pre_sql"` + PostSQL []string `yaml:"post_sql"` + // Incrustamos los settings para permitir los overrides + JobSettings `yaml:",inline"` +} + +type Config struct { + MaxParallelWorkers int `yaml:"max_parallel_workers"` + Defaults JobSettings `yaml:"defaults"` + Jobs []Job `yaml:"jobs"` +} + +func main() { + yamlFile, err := os.ReadFile("config.yaml") + if err != nil { + log.Fatalf("Error leyendo archivo: %v", err) + } + + var config Config + err = yaml.Unmarshal(yamlFile, &config) + if err != nil { + log.Fatalf("Error parseando YAML: %v", err) + } + + fmt.Printf("Configuración cargada. Trabajos: %d\n", len(config.Jobs)) + + for i, job := range config.Jobs { + jobPtr := &config.Jobs[i] + + if job.MaxExtractors == nil { + jobPtr.MaxExtractors = config.Defaults.MaxExtractors + } + if job.MaxLoaders == nil { + jobPtr.MaxLoaders = config.Defaults.MaxLoaders + } + if job.QueueSize == nil { + jobPtr.QueueSize = config.Defaults.QueueSize + } + if job.ChunkSize == nil { + jobPtr.ChunkSize = config.Defaults.ChunkSize + } + if job.ChunksPerBatch == nil { + jobPtr.ChunksPerBatch = config.Defaults.ChunksPerBatch + } + if job.TruncateTarget == nil { + jobPtr.TruncateTarget = config.Defaults.TruncateTarget + } + if job.TruncateMethod == nil { + jobPtr.TruncateMethod = config.Defaults.TruncateMethod + } + if job.Retry == nil { + jobPtr.Retry = config.Defaults.Retry + } + } + + printConfig(config) +} + +func printConfig(config Config) { + fmt.Println("Max parallel workers: ", config.MaxParallelWorkers) + + fmt.Println("Defaults:") + fmt.Printf("\tMaxExtractors: %v\n", *config.Defaults.MaxExtractors) + fmt.Printf("\tMaxLoaders: %v\n", *config.Defaults.MaxLoaders) + fmt.Printf("\tQueueSize: %v\n", *config.Defaults.QueueSize) + fmt.Printf("\tChunkSize: %v\n", *config.Defaults.ChunkSize) + fmt.Printf("\tChunksPerBatch: %v\n", *config.Defaults.ChunksPerBatch) + fmt.Printf("\tTruncateTarget: %v\n", *config.Defaults.TruncateTarget) + fmt.Printf("\tTruncateMethod: %v\n", *config.Defaults.TruncateMethod) + fmt.Printf("\tRetry: %v\n", *config.Defaults.Retry) + + fmt.Println("Jobs:") + for i, job := range config.Jobs { + fmt.Printf("Job Name: %v\n", job.Name) + fmt.Printf("\tEnabled: %v\n", job.Enabled) + fmt.Printf("\tSource: %v\n", job.Source) + fmt.Printf("\tTarget: %v\n", job.Target) + fmt.Printf("\tMaxExtractors: %v\n", *job.MaxExtractors) + fmt.Printf("\tMaxLoaders: %v\n", *job.MaxLoaders) + fmt.Printf("\tQueueSize: %v\n", *job.QueueSize) + fmt.Printf("\tChunkSize: %v\n", *job.ChunkSize) + fmt.Printf("\tChunksPerBatch: %v\n", *job.ChunksPerBatch) + fmt.Printf("\tTruncateTarget: %v\n", *job.TruncateTarget) + fmt.Printf("\tTruncateMethod: %v\n", *job.TruncateMethod) + fmt.Printf("\tRetry: %v\n", *job.Retry) + fmt.Printf("\tPreSQL: %v\n", job.PreSQL) + fmt.Printf("\tPostSQL: %v\n", job.PostSQL) + + if i >= 2 { + fmt.Println("Skipping remaining jobs...") + } + } +}