package config import ( "fmt" "os" "gopkg.in/yaml.v3" ) type RetryConfig struct { Attempts int `yaml:"attempts"` } type JobConfig struct { MaxExtractors int `yaml:"max_extractors"` MaxLoaders int `yaml:"max_loaders"` QueueSize int `yaml:"queue_size"` ChunkSize int `yaml:"chunk_size"` ChunksPerBatch int `yaml:"chunks_per_batch"` RowsPerBatch int64 TruncateTarget bool `yaml:"truncate_target"` TruncateMethod string `yaml:"truncate_method"` Retry RetryConfig `yaml:"retry"` } type TargetTableInfo struct { Schema string `yaml:"schema"` Table string `yaml:"table"` } type SourceTableInfo struct { Schema string `yaml:"schema"` Table string `yaml:"table"` PrimaryKey string `yaml:"primary_key"` } type Job struct { Name string `yaml:"name"` Enabled bool `yaml:"enabled"` SourceTable SourceTableInfo `yaml:"source"` TargetTable TargetTableInfo `yaml:"target"` PreSQL []string `yaml:"pre_sql"` PostSQL []string `yaml:"post_sql"` JobConfig `yaml:",inline"` } type MigrationConfig struct { MaxParallelWorkers int `yaml:"max_parallel_workers"` Defaults JobConfig `yaml:"defaults"` Jobs []Job `yaml:"jobs"` } type rawConfig struct { MaxParallelWorkers int `yaml:"max_parallel_workers"` Defaults JobConfig `yaml:"defaults"` Jobs []yaml.Node `yaml:"jobs"` } func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error { var raw rawConfig if err := value.Decode(&raw); err != nil { return err } c.MaxParallelWorkers = raw.MaxParallelWorkers c.Defaults = raw.Defaults c.Defaults.RowsPerBatch = int64(raw.Defaults.ChunkSize * raw.Defaults.ChunksPerBatch) for _, node := range raw.Jobs { job := Job{ JobConfig: raw.Defaults, } if err := node.Decode(&job); err != nil { return err } job.RowsPerBatch = int64(job.ChunkSize * job.ChunksPerBatch) c.Jobs = append(c.Jobs, job) } return nil } const defaultConfigFileName string = "config.yaml" func filenamesOrDefault(filenames []string) []string { if len(filenames) == 0 { return []string{defaultConfigFileName} } return filenames } func ReadMigrationConfig(filenames ...string) (MigrationConfig, error) { filenames = filenamesOrDefault(filenames) var data []byte var err error for _, filename := range filenames { data, err = os.ReadFile(filename) if err != nil { continue } break } if err != nil { return MigrationConfig{}, fmt.Errorf("Error reading config file: %v", err) } var config MigrationConfig if err := yaml.Unmarshal(data, &config); err != nil { return MigrationConfig{}, fmt.Errorf("Error parsing config file: %v", err) } return config, nil }