Kylesoda
74abf12dcf
Introduce a retry policy with exponential backoff and jitter for extractor and loader errors, with configurable max attempts and delay caps.
129 lines
3.2 KiB
Go
129 lines
3.2 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type RetryConfig struct {
|
|
Attempts int `yaml:"attempts"`
|
|
BaseDelayMs int `yaml:"base_delay_ms"`
|
|
MaxDelayMs int `yaml:"max_delay_ms"`
|
|
MaxJitterMs int `yaml:"max_jitter_ms"`
|
|
}
|
|
|
|
type JobConfig struct {
|
|
MaxExtractors int `yaml:"max_extractors"`
|
|
MaxLoaders int `yaml:"max_loaders"`
|
|
QueueSize int `yaml:"queue_size"`
|
|
BatchSize int `yaml:"batch_size"`
|
|
BatchesPerPartition int `yaml:"batches_per_partition"`
|
|
TruncateTarget bool `yaml:"truncate_target"`
|
|
TruncateMethod string `yaml:"truncate_method"`
|
|
Retry RetryConfig `yaml:"retry"`
|
|
RowsPerPartition int64
|
|
}
|
|
|
|
type TableInfo struct {
|
|
Schema string `yaml:"schema"`
|
|
Table string `yaml:"table"`
|
|
}
|
|
|
|
type TargetTableInfo struct {
|
|
TableInfo `yaml:",inline"`
|
|
}
|
|
|
|
type SourceTableInfo struct {
|
|
TableInfo `yaml:",inline"`
|
|
PrimaryKey string `yaml:"primary_key"`
|
|
}
|
|
|
|
type Job struct {
|
|
Name string `yaml:"name"`
|
|
Enabled bool `yaml:"enabled"`
|
|
SourceTable SourceTableInfo `yaml:"source"`
|
|
TargetTable TargetTableInfo `yaml:"target"`
|
|
PreSQL []string `yaml:"pre_sql"`
|
|
PostSQL []string `yaml:"post_sql"`
|
|
JobConfig `yaml:",inline"`
|
|
}
|
|
|
|
type MigrationConfig struct {
|
|
MaxParallelWorkers int `yaml:"max_parallel_workers"`
|
|
SourceDbType string `yaml:"source_db_type"`
|
|
TargetDbType string `yaml:"target_db_type"`
|
|
Defaults JobConfig `yaml:"defaults"`
|
|
Jobs []Job `yaml:"jobs"`
|
|
}
|
|
|
|
type rawConfig struct {
|
|
MaxParallelWorkers int `yaml:"max_parallel_workers"`
|
|
SourceDbType string `yaml:"source_db_type"`
|
|
TargetDbType string `yaml:"target_db_type"`
|
|
Defaults JobConfig `yaml:"defaults"`
|
|
Jobs []yaml.Node `yaml:"jobs"`
|
|
}
|
|
|
|
func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
|
|
var raw rawConfig
|
|
if err := value.Decode(&raw); err != nil {
|
|
return err
|
|
}
|
|
|
|
c.MaxParallelWorkers = raw.MaxParallelWorkers
|
|
c.Defaults = raw.Defaults
|
|
c.Defaults.RowsPerPartition = int64(raw.Defaults.BatchSize * raw.Defaults.BatchesPerPartition)
|
|
|
|
for _, node := range raw.Jobs {
|
|
job := Job{
|
|
JobConfig: raw.Defaults,
|
|
}
|
|
|
|
if err := node.Decode(&job); err != nil {
|
|
return err
|
|
}
|
|
|
|
job.RowsPerPartition = int64(job.BatchSize * job.BatchesPerPartition)
|
|
|
|
c.Jobs = append(c.Jobs, job)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
const defaultConfigFileName string = "config.yaml"
|
|
|
|
func filenamesOrDefault(filenames []string) []string {
|
|
if len(filenames) == 0 {
|
|
return []string{defaultConfigFileName}
|
|
}
|
|
return filenames
|
|
}
|
|
|
|
func ReadMigrationConfig(filenames ...string) (MigrationConfig, error) {
|
|
filenames = filenamesOrDefault(filenames)
|
|
var data []byte
|
|
var err error
|
|
|
|
for _, filename := range filenames {
|
|
data, err = os.ReadFile(filename)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
break
|
|
}
|
|
|
|
if err != nil {
|
|
return MigrationConfig{}, fmt.Errorf("Error reading config file: %v", err)
|
|
}
|
|
|
|
var config MigrationConfig
|
|
if err := yaml.Unmarshal(data, &config); err != nil {
|
|
return MigrationConfig{}, fmt.Errorf("Error parsing config file: %v", err)
|
|
}
|
|
|
|
return config, nil
|
|
}
|