refactor: update extractor interfaces to return row counts instead of using pointers for rows read
This commit is contained in:
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
|
||||
@@ -60,17 +59,17 @@ func (postgresEx *PostgresExtractor) ProcessPartition(
|
||||
partition models.Partition,
|
||||
indexPrimaryKey int,
|
||||
chBatchesOut chan<- models.Batch,
|
||||
rowsRead *int64,
|
||||
) error {
|
||||
) (int, error) {
|
||||
query := buildExtractQueryPostgres(tableInfo, columns)
|
||||
|
||||
if partition.ShouldUseRange {
|
||||
return errors.New("Batch config not yet supported")
|
||||
return 0, errors.New("Batch config not yet supported")
|
||||
}
|
||||
|
||||
rowsRead := 0
|
||||
rows, err := postgresEx.db.Query(ctx, query)
|
||||
if err != nil {
|
||||
return &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
|
||||
return rowsRead, &custom_errors.ExtractorError{Partition: partition, HasLastId: false, Msg: err.Error()}
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
@@ -79,8 +78,9 @@ func (postgresEx *PostgresExtractor) ProcessPartition(
|
||||
for rows.Next() {
|
||||
values, err := rows.Values()
|
||||
if err != nil {
|
||||
return errors.New("Unexpected error reading rows from source")
|
||||
return rowsRead, errors.New("Unexpected error reading rows from source")
|
||||
}
|
||||
rowsRead++
|
||||
|
||||
batchRows = append(batchRows, values)
|
||||
|
||||
@@ -88,29 +88,26 @@ func (postgresEx *PostgresExtractor) ProcessPartition(
|
||||
select {
|
||||
case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
return rowsRead, ctx.Err()
|
||||
}
|
||||
|
||||
atomic.AddInt64(rowsRead, int64(len(batchRows)))
|
||||
batchRows = make([]models.UnknownRowValues, 0, batchSize)
|
||||
}
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return errors.New("Unexpected error reading rows from source")
|
||||
return rowsRead, errors.New("Unexpected error reading rows from source")
|
||||
}
|
||||
|
||||
if len(batchRows) > 0 {
|
||||
select {
|
||||
case chBatchesOut <- models.Batch{Id: uuid.New(), PartitionId: partition.Id, Rows: batchRows, RetryCounter: 0}:
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
return rowsRead, nil
|
||||
}
|
||||
|
||||
atomic.AddInt64(rowsRead, int64(len(batchRows)))
|
||||
}
|
||||
|
||||
return nil
|
||||
return rowsRead, nil
|
||||
}
|
||||
|
||||
func (postgresEx *PostgresExtractor) Exec(
|
||||
|
||||
Reference in New Issue
Block a user