kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: simplify batch processing by removing partition dependency and introducing batch accumulator

Browse Source
This commit is contained in:
Kylesoda
2026-05-11 00:38:42 -05:00
parent 16217f6ee2
commit ab9a2d8694
5 changed files with 81 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/app/etl/extractors/process.go
View File

@@ -90,7 +90,7 @@ func (ex *GenericExtractor) ProcessPartition(
return rowsRead, err
}
if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
if err := flush(ctx, batchSize, batchRows, chBatchesOut); err != nil {
return rowsRead, err
}
@@ -102,7 +102,7 @@ func (ex *GenericExtractor) ProcessPartition(
batchRows = append(batchRows, rowValues)
if len(batchRows) >= batchSize {
// logrus.Debugf("Batch size reached, flushing batch with %v rows (rowsRead=%v)", len(batchRows), rowsRead)
if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
if err := flush(ctx, batchSize, batchRows, chBatchesOut); err != nil {
// logrus.Warnf("Error flushing rows: %v", err)
return rowsRead, err
}
@@ -110,7 +110,7 @@ func (ex *GenericExtractor) ProcessPartition(
}
}
if err := flush(ctx, &partition, batchSize, batchRows, chBatchesOut); err != nil {
if err := flush(ctx, batchSize, batchRows, chBatchesOut); err != nil {
return rowsRead, err
}