feat: enhance concurrency management by adding WaitGroup support in extractors and loaders
This commit is contained in:
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -25,6 +26,7 @@ func extractorErrorHandler(
|
||||
chErrorsIn <-chan ExtractorError,
|
||||
chBatchesOut chan<- Batch,
|
||||
chJobErrorsOut chan<- JobError,
|
||||
wgActiveBatches *sync.WaitGroup,
|
||||
) {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
@@ -52,6 +54,8 @@ func extractorErrorHandler(
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
wgActiveBatches.Done()
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user