refactor: optimize row handling in mssql extractor and transformer
This commit is contained in:
@@ -89,14 +89,9 @@ func (mssqlEx *MssqlExtractor) Exec(
|
||||
batchRows := make([]models.UnknownRowValues, 0, batchSize)
|
||||
var rowsRead int64 = 0
|
||||
|
||||
rowValues := make([]any, len(columns))
|
||||
scanArgs := make([]any, len(columns))
|
||||
for i := range rowValues {
|
||||
scanArgs[i] = &rowValues[i]
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(scanArgs...); err != nil {
|
||||
values, err := rows.Values()
|
||||
if err != nil {
|
||||
if len(batchRows) == 0 {
|
||||
return rowsRead, err
|
||||
}
|
||||
@@ -110,7 +105,7 @@ func (mssqlEx *MssqlExtractor) Exec(
|
||||
}
|
||||
rowsRead++
|
||||
|
||||
batchRows = append(batchRows, rowValues)
|
||||
batchRows = append(batchRows, values)
|
||||
if len(batchRows) >= batchSize {
|
||||
if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
|
||||
return rowsRead, err
|
||||
|
||||
Reference in New Issue
Block a user