101 lines
1.8 KiB
Go
101 lines
1.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type ExtractorError struct {
|
|
Batch
|
|
LastId int64
|
|
HasLastId bool
|
|
Msg string
|
|
}
|
|
|
|
func (e *ExtractorError) Error() string {
|
|
return e.Msg
|
|
}
|
|
|
|
const maxRetryAttempts = 3
|
|
|
|
func extractorErrorHandler(
|
|
ctx context.Context,
|
|
chErrorsIn <-chan ExtractorError,
|
|
chBatchesOut chan<- Batch,
|
|
chJobErrorsOut chan<- JobError,
|
|
) {
|
|
for {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
|
|
case err, ok := <-chErrorsIn:
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
if err.RetryCounter >= maxRetryAttempts {
|
|
jobError := JobError{
|
|
ShouldCancelJob: false,
|
|
Msg: fmt.Sprintf("batch %v reached max retries (%d)", err.Id, maxRetryAttempts),
|
|
Prev: &err,
|
|
}
|
|
|
|
select {
|
|
case chJobErrorsOut <- jobError:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
|
|
newBatch := err.Batch
|
|
newBatch.RetryCounter++
|
|
|
|
if err.HasLastId {
|
|
newBatch.ParentId = err.Id
|
|
newBatch.Id = uuid.New()
|
|
newBatch.LowerLimit = err.LastId
|
|
newBatch.IsLowerLimitInclusive = false
|
|
}
|
|
|
|
select {
|
|
case chBatchesOut <- newBatch:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func ExtractorErrorFromLastRowMssql(lastRow UnknownRowValues, indexPrimaryKey int, batch *Batch, previousError error) ExtractorError {
|
|
lastIdRawValue := lastRow[indexPrimaryKey]
|
|
|
|
lastId, ok := ToInt64(lastIdRawValue)
|
|
if !ok {
|
|
currentBatch := *batch
|
|
currentBatch.RetryCounter = maxRetryAttempts
|
|
exError := ExtractorError{
|
|
Batch: currentBatch,
|
|
HasLastId: true,
|
|
Msg: fmt.Sprintf("Couldn't cast last id value as int: %s", previousError.Error()),
|
|
}
|
|
return exError
|
|
|
|
}
|
|
|
|
exError := ExtractorError{
|
|
Batch: *batch,
|
|
HasLastId: true,
|
|
LastId: lastId,
|
|
Msg: previousError.Error(),
|
|
}
|
|
return exError
|
|
}
|