refactor: update extractor methods to support FromJsonColumns; enhance data processing capabilities

2026-05-07 08:17:25 -05:00
parent 46ddd0d6b7
commit a216a8016f
6 changed files with 12 additions and 3 deletions
--- a/internal/app/etl/extractors/process.go
+++ b/internal/app/etl/extractors/process.go
@@ -47,6 +47,7 @@ func (ex *GenericExtractor) ProcessPartition(
 	partition models.Partition,
 	indexPrimaryKey int,
 	chBatchesOut chan<- models.Batch,
+	fromJsonColumns []config.FromJsonItem,
 ) (int64, error) {
 	query := dbwrapper.ExtractionQuery{
 		Schema:     tableInfo.Schema,
@@ -63,6 +64,7 @@ func (ex *GenericExtractor) ProcessPartition(
 			IsInclusive: partition.Range.IsMaxInclusive,
 			Value:       partition.Range.Max,
 		},
+		FromJsonColumns: fromJsonColumns,
 	}

 	// logrus.Debugf("Processing partition: %+v (%s.%s)", query, tableInfo.Schema, tableInfo.Table)