feat: implement job configuration structure and YAML parsing for migration jobs

internal/app/config/migration.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+)
+
+type RetryConfig struct {
+	Attempts int `yaml:"attempts"`
+}
+
+type JobConfig struct {
+	MaxExtractors  int         `yaml:"max_extractors"`
+	MaxLoaders     int         `yaml:"max_loaders"`
+	QueueSize      int         `yaml:"queue_size"`
+	ChunkSize      int         `yaml:"chunk_size"`
+	ChunksPerBatch int         `yaml:"chunks_per_batch"`
+	TruncateTarget bool        `yaml:"truncate_target"`
+	TruncateMethod string      `yaml:"truncate_method"`
+	Retry          RetryConfig `yaml:"retry"`
+}
+
+type SourceDbInfo struct {
+	Schema     string `yaml:"schema"`
+	Table      string `yaml:"table"`
+	PrimaryKey string `yaml:"primary_key"`
+}
+
+type TargetDbInfo struct {
+	Schema string `yaml:"schema"`
+	Table  string `yaml:"table"`
+}
+
+type Job struct {
+	Name      string       `yaml:"name"`
+	Enabled   bool         `yaml:"enabled"`
+	Source    SourceDbInfo `yaml:"source"`
+	Target    TargetDbInfo `yaml:"target"`
+	PreSQL    []string     `yaml:"pre_sql"`
+	PostSQL   []string     `yaml:"post_sql"`
+	JobConfig `yaml:",inline"`
+}
+
+type MigrationConfig struct {
+	MaxParallelWorkers int       `yaml:"max_parallel_workers"`
+	Defaults           JobConfig `yaml:"defaults"`
+	Jobs               []Job     `yaml:"jobs"`
+}
+
+type rawConfig struct {
+	maxParallelWorkers int         `yaml:"max_parallel_workers"`
+	defaults           JobConfig   `yaml:"defaults"`
+	jobs               []yaml.Node `yaml:"jobs"`
+}
+
+func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
+	var raw rawConfig
+	if err := value.Decode(&raw); err != nil {
+		return err
+	}
+
+	c.MaxParallelWorkers = raw.maxParallelWorkers
+	c.Defaults = raw.defaults
+
+	for _, node := range raw.jobs {
+		job := Job{
+			JobConfig: raw.defaults,
+		}
+
+		if err := node.Decode(&job); err != nil {
+			return err
+		}
+
+		c.Jobs = append(c.Jobs, job)
+	}
+
+	return nil
+}
+
+const defaultConfigFileName string = "config.yml"
+
+func filenamesOrDefault(filenames []string) []string {
+	if len(filenames) == 0 {
+		return []string{defaultConfigFileName}
+	}
+	return filenames
+}
+
+func ReadMigrationConfig(filenames ...string) (MigrationConfig, error) {
+	filenames = filenamesOrDefault(filenames)
+	var data []byte
+	var err error
+
+	for _, filename := range filenames {
+		data, err = os.ReadFile(filename)
+		if err != nil {
+			continue
+		}
+		break
+	}
+
+	if err != nil {
+		return MigrationConfig{}, fmt.Errorf("Error reading config file: %v", err)
+	}
+
+	var config MigrationConfig
+	if err := yaml.Unmarshal(data, &config); err != nil {
+		log.Fatalf("Error parsing config file: %v", err)
+	}
+
+	return config, nil
+}