70 lines
1.6 KiB
Go
70 lines
1.6 KiB
Go
package extractors
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors"
|
|
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
|
|
"github.com/google/uuid"
|
|
// "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
func (ex *GenericExtractor) ProcessPartitionWithRetries(
|
|
ctx context.Context,
|
|
tableInfo config.SourceTableInfo,
|
|
columns []models.ColumnType,
|
|
batchSize int,
|
|
partition models.Partition,
|
|
indexPrimaryKey int,
|
|
chBatchesOut chan<- models.Batch,
|
|
) (int64, error) {
|
|
var totalRowsRead int64
|
|
delay := time.Duration(time.Second * 1)
|
|
currentParitition := partition
|
|
|
|
for {
|
|
rowsRead, err := ex.ProcessPartition(
|
|
ctx,
|
|
tableInfo,
|
|
columns,
|
|
batchSize,
|
|
currentParitition,
|
|
indexPrimaryKey,
|
|
chBatchesOut,
|
|
)
|
|
// logrus.Debugf("Partition %v finished processing (%s.%s)", partition.Id, tableInfo.Schema, tableInfo.Table)
|
|
totalRowsRead += rowsRead
|
|
|
|
if err == nil {
|
|
return totalRowsRead, nil
|
|
}
|
|
|
|
if exError, ok := errors.AsType[*custom_errors.ExtractorError](err); ok {
|
|
currentParitition.RetryCounter++
|
|
|
|
if currentParitition.RetryCounter > 3 {
|
|
return totalRowsRead, &custom_errors.JobError{
|
|
Msg: fmt.Sprintf("Partition %v reached max retries", exError.Partition.Id),
|
|
Prev: err,
|
|
}
|
|
}
|
|
|
|
if exError.HasLastId {
|
|
currentParitition.ParentId = exError.Partition.Id
|
|
currentParitition.Id = uuid.New()
|
|
currentParitition.Range.Min = exError.LastId
|
|
currentParitition.Range.IsMinInclusive = false
|
|
}
|
|
|
|
time.Sleep(delay)
|
|
continue
|
|
}
|
|
|
|
return totalRowsRead, err
|
|
}
|
|
}
|