feat: integrate azure storage handling in migration pipeline
Stream transformer output to an Azure blob container when a job declares ToStorage, with per-job prefix and credential configuration.
This commit is contained in:
@@ -14,6 +14,16 @@ type RetryConfig struct {
|
||||
MaxJitterMs int `yaml:"max_jitter_ms"`
|
||||
}
|
||||
|
||||
type ToStorageColumnConfig struct {
|
||||
Source string `yaml:"source"`
|
||||
Target string `yaml:"target"`
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type ToStorageConfig struct {
|
||||
Columns []ToStorageColumnConfig `yaml:"columns"`
|
||||
}
|
||||
|
||||
type JobConfig struct {
|
||||
MaxExtractors int `yaml:"max_extractors"`
|
||||
MaxLoaders int `yaml:"max_loaders"`
|
||||
@@ -26,6 +36,7 @@ type JobConfig struct {
|
||||
MaxChunkErrors int `yaml:"max_chunk_errors"`
|
||||
Retry RetryConfig `yaml:"retry"`
|
||||
RowsPerPartition int64
|
||||
ToStorage ToStorageConfig `yaml:"to_storage"`
|
||||
}
|
||||
|
||||
type TableInfo struct {
|
||||
@@ -33,29 +44,31 @@ type TableInfo struct {
|
||||
Table string `yaml:"table"`
|
||||
}
|
||||
|
||||
type TargetTableInfo struct {
|
||||
TableInfo `yaml:",inline"`
|
||||
}
|
||||
|
||||
type SourceTableInfo struct {
|
||||
TableInfo `yaml:",inline"`
|
||||
PrimaryKey string `yaml:"primary_key"`
|
||||
}
|
||||
|
||||
type TargetTableInfo struct {
|
||||
TableInfo `yaml:",inline"`
|
||||
PreSQL []string `yaml:"pre_sql"`
|
||||
PostSQL []string `yaml:"post_sql"`
|
||||
}
|
||||
|
||||
type RangeConfig struct {
|
||||
Min int64 `yaml:"min"`
|
||||
Max int64 `yaml:"max"`
|
||||
IsMinInclusive bool `yaml:"is_min_inclusive"`
|
||||
IsMaxInclusive bool `yaml:"is_max_inclusive"`
|
||||
}
|
||||
|
||||
type Job struct {
|
||||
Name string `yaml:"name"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
SourceTable SourceTableInfo `yaml:"source"`
|
||||
TargetTable TargetTableInfo `yaml:"target"`
|
||||
PreSQL []string `yaml:"pre_sql"`
|
||||
PostSQL []string `yaml:"post_sql"`
|
||||
JobConfig `yaml:",inline"`
|
||||
Range struct {
|
||||
Min int64 `yaml:"min"`
|
||||
Max int64 `yaml:"max"`
|
||||
IsMinInclusive bool `yaml:"is_min_inclusive"`
|
||||
IsMaxInclusive bool `yaml:"is_max_inclusive"`
|
||||
}
|
||||
Range RangeConfig `yaml:"range"`
|
||||
}
|
||||
|
||||
type MigrationConfig struct {
|
||||
|
||||
Reference in New Issue
Block a user