refactor: optimize row handling in mssql extractor and transformer

This commit is contained in:
2026-04-17 01:10:45 -05:00
parent 46597c4ffd
commit ec96532d04
3 changed files with 8 additions and 11 deletions

View File

@@ -68,7 +68,7 @@ func Consume(
wgActivePartitions.Done()
if rowsReadResult > 0 {
atomic.AddInt64(rowsRead, int64(rowsReadResult))
atomic.AddInt64(rowsRead, rowsReadResult)
}
if err != nil {
@@ -86,8 +86,6 @@ func Consume(
case chErrorsOut <- custom_errors.JobError{ShouldCancelJob: false, Msg: err.Error(), Prev: err}:
}
}
continue
}
}
}

View File

@@ -89,14 +89,9 @@ func (mssqlEx *MssqlExtractor) Exec(
batchRows := make([]models.UnknownRowValues, 0, batchSize)
var rowsRead int64 = 0
rowValues := make([]any, len(columns))
scanArgs := make([]any, len(columns))
for i := range rowValues {
scanArgs[i] = &rowValues[i]
}
for rows.Next() {
if err := rows.Scan(scanArgs...); err != nil {
values, err := rows.Values()
if err != nil {
if len(batchRows) == 0 {
return rowsRead, err
}
@@ -110,7 +105,7 @@ func (mssqlEx *MssqlExtractor) Exec(
}
rowsRead++
batchRows = append(batchRows, rowValues)
batchRows = append(batchRows, values)
if len(batchRows) >= batchSize {
if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
return rowsRead, err

View File

@@ -74,6 +74,10 @@ func (mssqlTr *MssqlTransformer) ProcessBatch(
}
}
if rowValues == nil {
continue
}
for _, task := range transformationPlan {
val := rowValues[task.Index]
if val == nil {