kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: refactor migration job structure to use SourceTableInfo and TargetTableInfo for improved configuration handling

Browse Source
This commit is contained in:
Kylesoda
2026-04-09 19:20:50 -05:00
parent e8ace6ecf9
commit 524d892a60
9 changed files with 118 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
internal/app/config/migration.go
View File

@@ -4,7 +4,6 @@ import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)
@@ -13,35 +12,36 @@ type RetryConfig struct {
}
type JobConfig struct {
MaxExtractors int `yaml:"max_extractors"`
MaxLoaders int `yaml:"max_loaders"`
QueueSize int `yaml:"queue_size"`
ChunkSize int `yaml:"chunk_size"`
ChunksPerBatch int `yaml:"chunks_per_batch"`
MaxExtractors int `yaml:"max_extractors"`
MaxLoaders int `yaml:"max_loaders"`
QueueSize int `yaml:"queue_size"`
ChunkSize int `yaml:"chunk_size"`
ChunksPerBatch int `yaml:"chunks_per_batch"`
RowsPerBatch int64
TruncateTarget bool `yaml:"truncate_target"`
TruncateMethod string `yaml:"truncate_method"`
Retry RetryConfig `yaml:"retry"`
}
type SourceDbInfo struct {
type TargetTableInfo struct {
Schema string `yaml:"schema"`
Table string `yaml:"table"`
}
type SourceTableInfo struct {
Schema string `yaml:"schema"`
Table string `yaml:"table"`
PrimaryKey string `yaml:"primary_key"`
}
type TargetDbInfo struct {
Schema string `yaml:"schema"`
Table string `yaml:"table"`
}
type Job struct {
Name string `yaml:"name"`
Enabled bool `yaml:"enabled"`
Source SourceDbInfo `yaml:"source"`
Target TargetDbInfo `yaml:"target"`
PreSQL []string `yaml:"pre_sql"`
PostSQL []string `yaml:"post_sql"`
JobConfig `yaml:",inline"`
Name string `yaml:"name"`
Enabled bool `yaml:"enabled"`
SourceTable SourceTableInfo `yaml:"source"`
TargetTable TargetTableInfo `yaml:"target"`
PreSQL []string `yaml:"pre_sql"`
PostSQL []string `yaml:"post_sql"`
JobConfig `yaml:",inline"`
}
type MigrationConfig struct {
@@ -51,9 +51,9 @@ type MigrationConfig struct {
}
type rawConfig struct {
maxParallelWorkers int `yaml:"max_parallel_workers"`
defaults JobConfig `yaml:"defaults"`
jobs []yaml.Node `yaml:"jobs"`
MaxParallelWorkers int `yaml:"max_parallel_workers"`
Defaults JobConfig `yaml:"defaults"`
Jobs []yaml.Node `yaml:"jobs"`
}
func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
@@ -62,25 +62,28 @@ func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
return err
}
c.MaxParallelWorkers = raw.maxParallelWorkers
c.Defaults = raw.defaults
c.MaxParallelWorkers = raw.MaxParallelWorkers
c.Defaults = raw.Defaults
c.Defaults.RowsPerBatch = int64(raw.Defaults.ChunkSize * raw.Defaults.ChunksPerBatch)
for _, node := range raw.jobs {
for _, node := range raw.Jobs {
job := Job{
JobConfig: raw.defaults,
JobConfig: raw.Defaults,
}
if err := node.Decode(&job); err != nil {
return err
}
job.RowsPerBatch = int64(job.ChunkSize * job.ChunksPerBatch)
c.Jobs = append(c.Jobs, job)
}
return nil
}
const defaultConfigFileName string = "config.yml"
const defaultConfigFileName string = "config.yaml"
func filenamesOrDefault(filenames []string) []string {
if len(filenames) == 0 {
@@ -108,7 +111,7 @@ func ReadMigrationConfig(filenames ...string) (MigrationConfig, error) {
var config MigrationConfig
if err := yaml.Unmarshal(data, &config); err != nil {
log.Fatalf("Error parsing config file: %v", err)
return MigrationConfig{}, fmt.Errorf("Error parsing config file: %v", err)
}
return config, nil