refactor: rename Batch to Partition in error handling and processing functions for consistency
This commit is contained in:
@@ -34,18 +34,18 @@ func mapSlice[T any, V any](input []T, mapper func(T) V) []V {
|
||||
return result
|
||||
}
|
||||
|
||||
func (postgresLd *PostgresLoader) ProcessChunk(
|
||||
func (postgresLd *PostgresLoader) ProcessBatch(
|
||||
ctx context.Context,
|
||||
tableInfo config.TargetTableInfo,
|
||||
colNames []string,
|
||||
chunk models.Batch,
|
||||
batch models.Batch,
|
||||
) (int, error) {
|
||||
tableId := pgx.Identifier{tableInfo.Schema, tableInfo.Table}
|
||||
_, err := postgresLd.db.CopyFrom(
|
||||
ctx,
|
||||
tableId,
|
||||
colNames,
|
||||
pgx.CopyFromRows(chunk.Data),
|
||||
pgx.CopyFromRows(batch.Rows),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
@@ -60,20 +60,20 @@ func (postgresLd *PostgresLoader) ProcessChunk(
|
||||
}
|
||||
}
|
||||
|
||||
return 0, &custom_errors.LoaderError{Batch: chunk, Msg: err.Error()}
|
||||
return 0, &custom_errors.LoaderError{Batch: batch, Msg: err.Error()}
|
||||
}
|
||||
|
||||
return len(chunk.Data), nil
|
||||
return len(batch.Rows), nil
|
||||
}
|
||||
|
||||
func (postgresLd *PostgresLoader) Exec(
|
||||
ctx context.Context,
|
||||
tableInfo config.TargetTableInfo,
|
||||
columns []models.ColumnType,
|
||||
chChunksIn <-chan models.Batch,
|
||||
chBatchesIn <-chan models.Batch,
|
||||
chErrorsOut chan<- custom_errors.LoaderError,
|
||||
chJobErrorsOut chan<- custom_errors.JobError,
|
||||
wgActiveChunks *sync.WaitGroup,
|
||||
wgActiveBatches *sync.WaitGroup,
|
||||
rowsLoaded *int64,
|
||||
) {
|
||||
colNames := mapSlice(columns, func(col models.ColumnType) string {
|
||||
@@ -88,12 +88,12 @@ func (postgresLd *PostgresLoader) Exec(
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case chunk, ok := <-chChunksIn:
|
||||
case batch, ok := <-chBatchesIn:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
processedRows, err := postgresLd.ProcessChunk(ctx, tableInfo, colNames, chunk)
|
||||
processedRows, err := postgresLd.ProcessBatch(ctx, tableInfo, colNames, batch)
|
||||
|
||||
if err != nil {
|
||||
var ldError *custom_errors.LoaderError
|
||||
@@ -117,7 +117,7 @@ func (postgresLd *PostgresLoader) Exec(
|
||||
return
|
||||
}
|
||||
|
||||
wgActiveChunks.Done()
|
||||
wgActiveBatches.Done()
|
||||
atomic.AddInt64(rowsLoaded, int64(processedRows))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user