diff --git a/go.mod b/go.mod index 8d3a13e..af6b560 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,13 @@ go 1.25.7 require ( github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4 - github.com/goccy/go-yaml v1.19.2 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.9.1 github.com/joho/godotenv v1.5.1 github.com/microsoft/go-mssqldb v1.9.8 github.com/sirupsen/logrus v1.9.4 github.com/twpayne/go-geom v1.6.1 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -19,6 +19,8 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/sync v0.19.0 // indirect diff --git a/go.sum b/go.sum index 33febce..7fecac7 100644 --- a/go.sum +++ b/go.sum @@ -16,13 +16,12 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4 h1:4vH4+3zfwZTqoJEFw7DsTaH1V8jgVwnyeDvNi2TxzAc= github.com/gaspardle/go-mssqlclrgeo v0.0.0-20160129143314-97ceabf987a4/go.mod h1:jlB0I5BIfcJBGdV6rRGPthSBfeY86RGkSAwcsldbHJc= -github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= -github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= @@ -43,6 +42,10 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/microsoft/go-mssqldb v1.9.8 h1:d4IFMvF/o+HdpXUqbBfzHvn/NlFA75YGcfHUUvDFJEM= @@ -51,6 +54,8 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= @@ -73,6 +78,8 @@ golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/config/migration.go b/internal/app/config/migration.go new file mode 100644 index 0000000..072c0b0 --- /dev/null +++ b/internal/app/config/migration.go @@ -0,0 +1,115 @@ +package config + +import ( + "fmt" + "os" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +type RetryConfig struct { + Attempts int `yaml:"attempts"` +} + +type JobConfig struct { + MaxExtractors int `yaml:"max_extractors"` + MaxLoaders int `yaml:"max_loaders"` + QueueSize int `yaml:"queue_size"` + ChunkSize int `yaml:"chunk_size"` + ChunksPerBatch int `yaml:"chunks_per_batch"` + TruncateTarget bool `yaml:"truncate_target"` + TruncateMethod string `yaml:"truncate_method"` + Retry RetryConfig `yaml:"retry"` +} + +type SourceDbInfo struct { + Schema string `yaml:"schema"` + Table string `yaml:"table"` + PrimaryKey string `yaml:"primary_key"` +} + +type TargetDbInfo struct { + Schema string `yaml:"schema"` + Table string `yaml:"table"` +} + +type Job struct { + Name string `yaml:"name"` + Enabled bool `yaml:"enabled"` + Source SourceDbInfo `yaml:"source"` + Target TargetDbInfo `yaml:"target"` + PreSQL []string `yaml:"pre_sql"` + PostSQL []string `yaml:"post_sql"` + JobConfig `yaml:",inline"` +} + +type MigrationConfig struct { + MaxParallelWorkers int `yaml:"max_parallel_workers"` + Defaults JobConfig `yaml:"defaults"` + Jobs []Job `yaml:"jobs"` +} + +type rawConfig struct { + maxParallelWorkers int `yaml:"max_parallel_workers"` + defaults JobConfig `yaml:"defaults"` + jobs []yaml.Node `yaml:"jobs"` +} + +func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error { + var raw rawConfig + if err := value.Decode(&raw); err != nil { + return err + } + + c.MaxParallelWorkers = raw.maxParallelWorkers + c.Defaults = raw.defaults + + for _, node := range raw.jobs { + job := Job{ + JobConfig: raw.defaults, + } + + if err := node.Decode(&job); err != nil { + return err + } + + c.Jobs = append(c.Jobs, job) + } + + return nil +} + +const defaultConfigFileName string = "config.yml" + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{defaultConfigFileName} + } + return filenames +} + +func ReadMigrationConfig(filenames ...string) (MigrationConfig, error) { + filenames = filenamesOrDefault(filenames) + var data []byte + var err error + + for _, filename := range filenames { + data, err = os.ReadFile(filename) + if err != nil { + continue + } + break + } + + if err != nil { + return MigrationConfig{}, fmt.Errorf("Error reading config file: %v", err) + } + + var config MigrationConfig + if err := yaml.Unmarshal(data, &config); err != nil { + log.Fatalf("Error parsing config file: %v", err) + } + + return config, nil +} diff --git a/scripts/config-parser/main.go b/scripts/config-parser/main.go index 9b6e0e8..92f4c9f 100644 --- a/scripts/config-parser/main.go +++ b/scripts/config-parser/main.go @@ -1,128 +1,77 @@ package main import ( - "fmt" - "log" - "os" - - "github.com/goccy/go-yaml" + "gopkg.in/yaml.v3" ) type RetryConfig struct { Attempts int `yaml:"attempts"` } -type DBInfo struct { - Schema string `yaml:"schema"` - Table string `yaml:"table"` - PrimaryKey string `yaml:"primary_key,omitempty"` +type JobConfig struct { + MaxExtractors int `yaml:"max_extractors"` + MaxLoaders int `yaml:"max_loaders"` + QueueSize int `yaml:"queue_size"` + ChunkSize int `yaml:"chunk_size"` + ChunksPerBatch int `yaml:"chunks_per_batch"` + TruncateTarget bool `yaml:"truncate_target"` + TruncateMethod string `yaml:"truncate_method"` + Retry RetryConfig `yaml:"retry"` } -type JobSettings struct { - MaxExtractors *int `yaml:"max_extractors"` - MaxLoaders *int `yaml:"max_loaders"` - QueueSize *int `yaml:"queue_size"` - ChunkSize *int `yaml:"chunk_size"` - ChunksPerBatch *int `yaml:"chunks_per_batch"` - TruncateTarget *bool `yaml:"truncate_target"` - TruncateMethod *string `yaml:"truncate_method"` - Retry *RetryConfig `yaml:"retry"` +type SourceDbInfo struct { + Schema string `yaml:"schema"` + Table string `yaml:"table"` + PrimaryKey string `yaml:"primary_key"` +} + +type TargetDbInfo struct { + Schema string `yaml:"schema"` + Table string `yaml:"table"` } type Job struct { - Name string `yaml:"name"` - Enabled bool `yaml:"enabled"` - Source DBInfo `yaml:"source"` - Target DBInfo `yaml:"target"` - PreSQL []string `yaml:"pre_sql"` - PostSQL []string `yaml:"post_sql"` - JobSettings `yaml:",inline"` + Name string `yaml:"name"` + Enabled bool `yaml:"enabled"` + Source SourceDbInfo `yaml:"source"` + Target TargetDbInfo `yaml:"target"` + PreSQL []string `yaml:"pre_sql"` + PostSQL []string `yaml:"post_sql"` + JobConfig `yaml:",inline"` } -type Config struct { - MaxParallelWorkers int `yaml:"max_parallel_workers"` - Defaults JobSettings `yaml:"defaults"` - Jobs []Job `yaml:"jobs"` +type MigrationConfig struct { + MaxParallelWorkers int `yaml:"max_parallel_workers"` + Defaults JobConfig `yaml:"defaults"` + Jobs []Job `yaml:"jobs"` } -func main() { - yamlFile, err := os.ReadFile("config.yaml") - if err != nil { - log.Fatalf("Error leyendo archivo: %v", err) - } - - var config Config - err = yaml.Unmarshal(yamlFile, &config) - if err != nil { - log.Fatalf("Error parseando YAML: %v", err) - } - - fmt.Printf("Configuración cargada. Trabajos: %d\n", len(config.Jobs)) - - for i, job := range config.Jobs { - jobPtr := &config.Jobs[i] - - if job.MaxExtractors == nil { - jobPtr.MaxExtractors = config.Defaults.MaxExtractors - } - if job.MaxLoaders == nil { - jobPtr.MaxLoaders = config.Defaults.MaxLoaders - } - if job.QueueSize == nil { - jobPtr.QueueSize = config.Defaults.QueueSize - } - if job.ChunkSize == nil { - jobPtr.ChunkSize = config.Defaults.ChunkSize - } - if job.ChunksPerBatch == nil { - jobPtr.ChunksPerBatch = config.Defaults.ChunksPerBatch - } - if job.TruncateTarget == nil { - jobPtr.TruncateTarget = config.Defaults.TruncateTarget - } - if job.TruncateMethod == nil { - jobPtr.TruncateMethod = config.Defaults.TruncateMethod - } - if job.Retry == nil { - jobPtr.Retry = config.Defaults.Retry - } - } - - printConfig(config) +type rawConfig struct { + maxParallelWorkers int `yaml:"max_parallel_workers"` + defaults JobConfig `yaml:"defaults"` + jobs []yaml.Node `yaml:"jobs"` } -func printConfig(config Config) { - fmt.Println("Max parallel workers: ", config.MaxParallelWorkers) - - fmt.Println("Defaults:") - fmt.Printf("\tMaxExtractors: %v\n", *config.Defaults.MaxExtractors) - fmt.Printf("\tMaxLoaders: %v\n", *config.Defaults.MaxLoaders) - fmt.Printf("\tQueueSize: %v\n", *config.Defaults.QueueSize) - fmt.Printf("\tChunkSize: %v\n", *config.Defaults.ChunkSize) - fmt.Printf("\tChunksPerBatch: %v\n", *config.Defaults.ChunksPerBatch) - fmt.Printf("\tTruncateTarget: %v\n", *config.Defaults.TruncateTarget) - fmt.Printf("\tTruncateMethod: %v\n", *config.Defaults.TruncateMethod) - fmt.Printf("\tRetry: %+v\n", *config.Defaults.Retry) - - fmt.Println("Jobs:") - for i, job := range config.Jobs { - fmt.Printf("Job Name: %v\n", job.Name) - fmt.Printf("\tEnabled: %v\n", job.Enabled) - fmt.Printf("\tSource: %+v\n", job.Source) - fmt.Printf("\tTarget: %+v\n", job.Target) - fmt.Printf("\tMaxExtractors: %v\n", *job.MaxExtractors) - fmt.Printf("\tMaxLoaders: %v\n", *job.MaxLoaders) - fmt.Printf("\tQueueSize: %v\n", *job.QueueSize) - fmt.Printf("\tChunkSize: %v\n", *job.ChunkSize) - fmt.Printf("\tChunksPerBatch: %v\n", *job.ChunksPerBatch) - fmt.Printf("\tTruncateTarget: %v\n", *job.TruncateTarget) - fmt.Printf("\tTruncateMethod: %v\n", *job.TruncateMethod) - fmt.Printf("\tRetry: %+v\n", *job.Retry) - fmt.Printf("\tPreSQL: %v\n", job.PreSQL) - fmt.Printf("\tPostSQL: %v\n", job.PostSQL) - - if i >= 2 { - fmt.Println("Skipping remaining jobs...") - } +func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error { + var raw rawConfig + if err := value.Decode(&raw); err != nil { + return err } + + c.MaxParallelWorkers = raw.maxParallelWorkers + c.Defaults = raw.defaults + + for _, node := range raw.jobs { + job := Job{ + JobConfig: raw.defaults, + } + + if err := node.Decode(&job); err != nil { + return err + } + + c.Jobs = append(c.Jobs, job) + } + + return nil }