refactor: enhance config handling and unify main entrypoint
Unify configuration parsing logic across entrypoints and clean up legacy script-side config paths.
This commit is contained in:
@@ -22,6 +22,7 @@ func (ex *GenericExtractor) ProcessPartitionWithRetries(
|
||||
indexPrimaryKey int,
|
||||
retryConfig config.RetryConfig,
|
||||
chBatchesOut chan<- models.Batch,
|
||||
fromJsonColumns []config.FromJsonItem,
|
||||
) (int64, error) {
|
||||
var totalRowsRead int64
|
||||
currentParitition := partition
|
||||
@@ -35,6 +36,7 @@ func (ex *GenericExtractor) ProcessPartitionWithRetries(
|
||||
currentParitition,
|
||||
indexPrimaryKey,
|
||||
chBatchesOut,
|
||||
fromJsonColumns,
|
||||
)
|
||||
// logrus.Debugf("Partition %v finished processing (%s.%s)", partition.Id, tableInfo.Schema, tableInfo.Table)
|
||||
totalRowsRead += rowsRead
|
||||
@@ -48,7 +50,7 @@ func (ex *GenericExtractor) ProcessPartitionWithRetries(
|
||||
|
||||
if currentParitition.RetryCounter >= retryConfig.Attempts {
|
||||
return totalRowsRead, &custom_errors.JobError{
|
||||
Msg: fmt.Sprintf("Partition %v reached max retries", exError.Partition.Id),
|
||||
Msg: fmt.Sprintf("Partition %v reached max retries (%d)", currentParitition.Id, currentParitition.RetryCounter),
|
||||
Prev: err,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user