refactor: update extractor methods to support FromJsonColumns; enhance data processing capabilities

2026-05-07 08:17:25 -05:00
parent 46ddd0d6b7
commit a216a8016f
6 changed files with 12 additions and 3 deletions
--- a/internal/app/etl/extractors/process-with-retries.go
+++ b/internal/app/etl/extractors/process-with-retries.go
@@ -22,6 +22,7 @@ func (ex *GenericExtractor) ProcessPartitionWithRetries(
 	indexPrimaryKey int,
 	retryConfig config.RetryConfig,
 	chBatchesOut chan<- models.Batch,
+	fromJsonColumns []config.FromJsonItem,
 ) (int64, error) {
 	var totalRowsRead int64
 	currentParitition := partition
@@ -35,6 +36,7 @@ func (ex *GenericExtractor) ProcessPartitionWithRetries(
 			currentParitition,
 			indexPrimaryKey,
 			chBatchesOut,
+			fromJsonColumns,
 		)
 		// logrus.Debugf("Partition %v finished processing (%s.%s)", partition.Id, tableInfo.Schema, tableInfo.Table)
 		totalRowsRead += rowsRead