feat: enhance concurrency management by adding WaitGroup support in extractors and loaders

This commit is contained in:
2026-04-09 00:22:30 -05:00
parent dc632361e5
commit 51480015ba
8 changed files with 82 additions and 68 deletions

View File

@@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"sync"
"github.com/google/uuid"
)
@@ -25,6 +26,7 @@ func extractorErrorHandler(
chErrorsIn <-chan ExtractorError,
chBatchesOut chan<- Batch,
chJobErrorsOut chan<- JobError,
wgActiveBatches *sync.WaitGroup,
) {
for {
if ctx.Err() != nil {
@@ -52,6 +54,8 @@ func extractorErrorHandler(
case <-ctx.Done():
return
}
wgActiveBatches.Done()
continue
}