feat: add ToStorage configuration to JobConfig for enhanced data handling

This commit is contained in:
2026-04-21 13:43:03 -05:00
parent cd2efb8692
commit bb7b35619a

View File

@@ -14,6 +14,16 @@ type RetryConfig struct {
MaxJitterMs int `yaml:"max_jitter_ms"` MaxJitterMs int `yaml:"max_jitter_ms"`
} }
type ToStorageColumnConfig struct {
Source string `yaml:"source"`
Target string `yaml:"target"`
Mode string `yaml:"mode"`
}
type ToStorageConfig struct {
Columns []ToStorageColumnConfig `yaml:"columns"`
}
type JobConfig struct { type JobConfig struct {
MaxExtractors int `yaml:"max_extractors"` MaxExtractors int `yaml:"max_extractors"`
MaxLoaders int `yaml:"max_loaders"` MaxLoaders int `yaml:"max_loaders"`
@@ -26,6 +36,7 @@ type JobConfig struct {
MaxChunkErrors int `yaml:"max_chunk_errors"` MaxChunkErrors int `yaml:"max_chunk_errors"`
Retry RetryConfig `yaml:"retry"` Retry RetryConfig `yaml:"retry"`
RowsPerPartition int64 RowsPerPartition int64
ToStorage ToStorageConfig `yaml:"to_storage"`
} }
type TableInfo struct { type TableInfo struct {