refactor: rename Batch to Partition across pipeline

Rename internal types and channels from Batch to Partition for consistency with the data model.

internal/app/config/main.go

@@ -8,10 +8,8 @@ import (
 )
 
 type appConfig struct {
-	SourceDbUrl  string
-	SourceDbType string
-	TargetDbUrl  string
-	TargetDbType string
+	SourceDbUrl string
+	TargetDbUrl string
 }
 
 func loadEnv() {
@@ -29,26 +27,14 @@ func getAppConfig() appConfig {
 		log.Fatal("SOURCE_DB_URL environment variable not set")
 	}
 
-	sourceDbType := os.Getenv("SOURCE_DB_TYPE")
-	if sourceDbType == "" {
-		log.Fatal("SOURCE_DB_TYPE environment variable not set")
-	}
-
 	targetDbUrl := os.Getenv("TARGET_DB_URL")
 	if targetDbUrl == "" {
 		log.Fatal("TARGET_DB_URL environment variable not set")
 	}
 
-	targetDbType := os.Getenv("TARGET_DB_TYPE")
-	if targetDbType == "" {
-		log.Fatal("TARGET_DB_TYPE environment variable not set")
-	}
-
 	return appConfig{
-		SourceDbUrl:  sourceDbUrl,
-		SourceDbType: sourceDbType,
-		TargetDbUrl:  targetDbUrl,
-		TargetDbType: targetDbType,
+		SourceDbUrl: sourceDbUrl,
+		TargetDbUrl: targetDbUrl,
 	}
 }
 

internal/app/config/migration.go

@@ -0,0 +1,125 @@
+package config
+
+import (
+	"fmt"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type RetryConfig struct {
+	Attempts int `yaml:"attempts"`
+}
+
+type JobConfig struct {
+	MaxExtractors  int `yaml:"max_extractors"`
+	MaxLoaders     int `yaml:"max_loaders"`
+	QueueSize      int `yaml:"queue_size"`
+	ChunkSize      int `yaml:"chunk_size"`
+	ChunksPerBatch int `yaml:"chunks_per_batch"`
+	RowsPerBatch   int64
+	TruncateTarget bool        `yaml:"truncate_target"`
+	TruncateMethod string      `yaml:"truncate_method"`
+	Retry          RetryConfig `yaml:"retry"`
+}
+
+type TableInfo struct {
+	Schema string `yaml:"schema"`
+	Table  string `yaml:"table"`
+}
+
+type TargetTableInfo struct {
+	TableInfo `yaml:",inline"`
+}
+
+type SourceTableInfo struct {
+	TableInfo  `yaml:",inline"`
+	PrimaryKey string `yaml:"primary_key"`
+}
+
+type Job struct {
+	Name        string          `yaml:"name"`
+	Enabled     bool            `yaml:"enabled"`
+	SourceTable SourceTableInfo `yaml:"source"`
+	TargetTable TargetTableInfo `yaml:"target"`
+	PreSQL      []string        `yaml:"pre_sql"`
+	PostSQL     []string        `yaml:"post_sql"`
+	JobConfig   `yaml:",inline"`
+}
+
+type MigrationConfig struct {
+	MaxParallelWorkers int       `yaml:"max_parallel_workers"`
+	SourceDbType       string    `yaml:"source_db_type"`
+	TargetDbType       string    `yaml:"target_db_type"`
+	Defaults           JobConfig `yaml:"defaults"`
+	Jobs               []Job     `yaml:"jobs"`
+}
+
+type rawConfig struct {
+	MaxParallelWorkers int         `yaml:"max_parallel_workers"`
+	SourceDbType       string      `yaml:"source_db_type"`
+	TargetDbType       string      `yaml:"target_db_type"`
+	Defaults           JobConfig   `yaml:"defaults"`
+	Jobs               []yaml.Node `yaml:"jobs"`
+}
+
+func (c *MigrationConfig) UnmarshalYAML(value *yaml.Node) error {
+	var raw rawConfig
+	if err := value.Decode(&raw); err != nil {
+		return err
+	}
+
+	c.MaxParallelWorkers = raw.MaxParallelWorkers
+	c.Defaults = raw.Defaults
+	c.Defaults.RowsPerBatch = int64(raw.Defaults.ChunkSize * raw.Defaults.ChunksPerBatch)
+
+	for _, node := range raw.Jobs {
+		job := Job{
+			JobConfig: raw.Defaults,
+		}
+
+		if err := node.Decode(&job); err != nil {
+			return err
+		}
+
+		job.RowsPerBatch = int64(job.ChunkSize * job.ChunksPerBatch)
+
+		c.Jobs = append(c.Jobs, job)
+	}
+
+	return nil
+}
+
+const defaultConfigFileName string = "config.yaml"
+
+func filenamesOrDefault(filenames []string) []string {
+	if len(filenames) == 0 {
+		return []string{defaultConfigFileName}
+	}
+	return filenames
+}
+
+func ReadMigrationConfig(filenames ...string) (MigrationConfig, error) {
+	filenames = filenamesOrDefault(filenames)
+	var data []byte
+	var err error
+
+	for _, filename := range filenames {
+		data, err = os.ReadFile(filename)
+		if err != nil {
+			continue
+		}
+		break
+	}
+
+	if err != nil {
+		return MigrationConfig{}, fmt.Errorf("Error reading config file: %v", err)
+	}
+
+	var config MigrationConfig
+	if err := yaml.Unmarshal(data, &config); err != nil {
+		return MigrationConfig{}, fmt.Errorf("Error parsing config file: %v", err)
+	}
+
+	return config, nil
+}