kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: rename Batch to Partition and update related types and channels for consistency

Browse Source
This commit is contained in:
Kylesoda
2026-04-11 00:09:28 -05:00
parent cd0e53b1d2
commit 9eb9821daf
11 changed files with 69 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/app/etl/extractors/mssql.go
View File

@@ -73,7 +73,7 @@ func buildExtractQueryMssql(
func extractorErrorFromLastRowMssql(
lastRow models.UnknownRowValues,
indexPrimaryKey int,
batch *models.Batch,
batch *models.Partition,
previousError error,
) *custom_errors.ExtractorError {
lastIdRawValue := lastRow[indexPrimaryKey]
@@ -103,9 +103,9 @@ func (mssqlEx *MssqlExtractor) ProcessBatch(
tableInfo config.SourceTableInfo,
columns []models.ColumnType,
chunkSize int,
batch models.Batch,
batch models.Partition,
indexPrimaryKey int,
chChunksOut chan<- models.Chunk,
chChunksOut chan<- models.Batch,
rowsRead *int64,
) error {
query := buildExtractQueryMssql(tableInfo, columns, batch.ShouldUseRange, batch.IsLowerLimitInclusive)
@@ -142,7 +142,7 @@ func (mssqlEx *MssqlExtractor) ProcessBatch(
lastRow := rowsChunk[len(rowsChunk)-1]
select {
case chChunksOut <- models.Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case chChunksOut <- models.Batch{Id: uuid.New(), PartitionId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case <-ctx.Done():
return nil
}
@@ -156,7 +156,7 @@ func (mssqlEx *MssqlExtractor) ProcessBatch(
if len(rowsChunk) >= chunkSize {
select {
case chChunksOut <- models.Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case chChunksOut <- models.Batch{Id: uuid.New(), PartitionId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case <-ctx.Done():
return nil
}
@@ -181,7 +181,7 @@ func (mssqlEx *MssqlExtractor) ProcessBatch(
if len(rowsChunk) > 0 {
select {
case chChunksOut <- models.Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case chChunksOut <- models.Batch{Id: uuid.New(), PartitionId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case <-ctx.Done():
return nil
}
@@ -197,8 +197,8 @@ func (mssqlEx *MssqlExtractor) Exec(
tableInfo config.SourceTableInfo,
columns []models.ColumnType,
chunkSize int,
chBatchesIn <-chan models.Batch,
chChunksOut chan<- models.Chunk,
chBatchesIn <-chan models.Partition,
chChunksOut chan<- models.Batch,
chErrorsOut chan<- custom_errors.ExtractorError,
chJobErrorsOut chan<- custom_errors.JobError,
wgActiveBatches *sync.WaitGroup,

12
internal/app/etl/extractors/postgres.go
View File

@@ -57,9 +57,9 @@ func (postgresEx *PostgresExtractor) ProcessBatch(
tableInfo config.SourceTableInfo,
columns []models.ColumnType,
chunkSize int,
batch models.Batch,
batch models.Partition,
indexPrimaryKey int,
chChunksOut chan<- models.Chunk,
chChunksOut chan<- models.Batch,
rowsRead *int64,
) error {
query := buildExtractQueryPostgres(tableInfo, columns)
@@ -86,7 +86,7 @@ func (postgresEx *PostgresExtractor) ProcessBatch(
if len(rowsChunk) >= chunkSize {
select {
case chChunksOut <- models.Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case chChunksOut <- models.Batch{Id: uuid.New(), PartitionId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case <-ctx.Done():
return nil
}
@@ -102,7 +102,7 @@ func (postgresEx *PostgresExtractor) ProcessBatch(
if len(rowsChunk) > 0 {
select {
case chChunksOut <- models.Chunk{Id: uuid.New(), BatchId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case chChunksOut <- models.Batch{Id: uuid.New(), PartitionId: batch.Id, Data: rowsChunk, RetryCounter: 0}:
case <-ctx.Done():
return nil
}
@@ -118,8 +118,8 @@ func (postgresEx *PostgresExtractor) Exec(
tableInfo config.SourceTableInfo,
columns []models.ColumnType,
chunkSize int,
chBatchesIn <-chan models.Batch,
chChunksOut chan<- models.Chunk,
chBatchesIn <-chan models.Partition,
chChunksOut chan<- models.Batch,
chErrorsOut chan<- custom_errors.ExtractorError,
chJobErrorsOut chan<- custom_errors.JobError,
wgActiveBatches *sync.WaitGroup,