feat: add database wrapper interfaces for mssql and postgres

Introduce a DB dialect interface and per-dialect wrappers that expose connection, query and column type operations with pre/post SQL hooks.

internal/app/config/migration.go

@@ -8,9 +8,9 @@ import (
 )
 
 type RetryConfig struct {
-	Attempts int `yaml:"attempts"`
+	Attempts    int `yaml:"attempts"`
 	BaseDelayMs int `yaml:"base_delay_ms"`
-	MaxDelayMs int `yaml:"max_delay_ms"`
+	MaxDelayMs  int `yaml:"max_delay_ms"`
 	MaxJitterMs int `yaml:"max_jitter_ms"`
 }
 
@@ -22,6 +22,8 @@ type JobConfig struct {
 	BatchesPerPartition int         `yaml:"batches_per_partition"`
 	TruncateTarget      bool        `yaml:"truncate_target"`
 	TruncateMethod      string      `yaml:"truncate_method"`
+	MaxPartitionErrrors int         `yaml:"max_partition_errrors"`
+	MaxChunkErrors      int         `yaml:"max_chunk_errors"`
 	Retry               RetryConfig `yaml:"retry"`
 	RowsPerPartition    int64
 }

internal/app/custom_errors/extractor.error.go

@@ -24,11 +24,14 @@ func (e *ExtractorError) Error() string {
 func ExtractorErrorHandler(
 	ctx context.Context,
 	retryConfig config.RetryConfig,
+	maxPartitionErrors int,
 	chErrorsIn <-chan ExtractorError,
 	chPartitionsOut chan<- models.Partition,
 	chJobErrorsOut chan<- JobError,
 	wgActivePartitions *sync.WaitGroup,
 ) {
+	definitiveErrors := 0
+
 	for {
 		if ctx.Err() != nil {
 			return
@@ -45,6 +48,7 @@ func ExtractorErrorHandler(
 
 			if err.Partition.RetryCounter >= retryConfig.Attempts {
 				wgActivePartitions.Done()
+				definitiveErrors++
 				jobError := JobError{
 					ShouldCancelJob: false,
 					Msg:             fmt.Sprintf("Partition %v reached max retries (%d)", err.Partition.Id, retryConfig.Attempts),
@@ -57,6 +61,20 @@ func ExtractorErrorHandler(
 					return
 				}
 
+				if maxPartitionErrors > 0 && definitiveErrors >= maxPartitionErrors {
+					fatalError := JobError{
+						ShouldCancelJob: true,
+						Msg:             fmt.Sprintf("Partition error limit reached (%d)", maxPartitionErrors),
+						Prev:            &err,
+					}
+
+					select {
+					case chJobErrorsOut <- fatalError:
+					case <-ctx.Done():
+						return
+					}
+				}
+
 				continue
 			} else {
 				jobError := JobError{

internal/app/custom_errors/loader.error.go

@@ -21,11 +21,14 @@ func (e *LoaderError) Error() string {
 func LoaderErrorHandler(
 	ctx context.Context,
 	retryConfig config.RetryConfig,
+	maxChunkErrors int,
 	chErrorsIn <-chan LoaderError,
 	chBatchesOut chan<- models.Batch,
 	chJobErrorsOut chan<- JobError,
 	wgActiveBatches *sync.WaitGroup,
 ) {
+	definitiveErrors := 0
+
 	for {
 		if ctx.Err() != nil {
 			return
@@ -42,6 +45,7 @@ func LoaderErrorHandler(
 
 			if err.Batch.RetryCounter >= retryConfig.Attempts {
 				wgActiveBatches.Done()
+				definitiveErrors++
 				jobError := JobError{
 					ShouldCancelJob: false,
 					Msg:             fmt.Sprintf("Batch %v reached max retries (%d)", err.Batch.Id, retryConfig.Attempts),
@@ -54,6 +58,20 @@ func LoaderErrorHandler(
 					return
 				}
 
+				if maxChunkErrors > 0 && definitiveErrors >= maxChunkErrors {
+					fatalError := JobError{
+						ShouldCancelJob: true,
+						Msg:             fmt.Sprintf("Chunk error limit reached (%d)", maxChunkErrors),
+						Prev:            &err,
+					}
+
+					select {
+					case chJobErrorsOut <- fatalError:
+					case <-ctx.Done():
+						return
+					}
+				}
+
 				continue
 			} else {
 				jobError := JobError{

internal/app/db/mssql.go

@@ -0,0 +1,30 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+)
+
+type MssqlDbWrapper struct {
+	db *sql.DB
+}
+
+func NewMssqlDbWrapper(db *sql.DB) DbWrapper {
+	return &MssqlDbWrapper{db: db}
+}
+
+func (wrapper *MssqlDbWrapper) Exec(ctx context.Context, query string, args ...any) (DbWrapperResult, error) {
+	result, execErr := wrapper.db.ExecContext(ctx, query, args...)
+	if execErr != nil {
+		return DbWrapperResult{}, execErr
+	}
+
+	affectedRows, err := result.RowsAffected()
+	if err != nil {
+		return DbWrapperResult{}, err
+	}
+
+	return DbWrapperResult{
+		AffectedRows: affectedRows,
+	}, nil
+}

internal/app/db/postgres.go

@@ -26,3 +26,22 @@ func Close(pool *pgxpool.Pool) {
 		pool.Close()
 	}
 }
+
+type PostgresDbWrapper struct {
+	db *pgxpool.Pool
+}
+
+func NewPostgresDbWrapper(db *pgxpool.Pool) DbWrapper {
+	return &PostgresDbWrapper{db: db}
+}
+
+func (wrapper *PostgresDbWrapper) Exec(ctx context.Context, query string, args ...any) (DbWrapperResult, error) {
+	result, err := wrapper.db.Exec(ctx, query, args...)
+	if err != nil {
+		return DbWrapperResult{}, err
+	}
+
+	return DbWrapperResult{
+		AffectedRows: result.RowsAffected(),
+	}, nil
+}

internal/app/db/types.go

@@ -0,0 +1,11 @@
+package db
+
+import "context"
+
+type DbWrapperResult struct {
+	AffectedRows int64
+}
+
+type DbWrapper interface {
+	Exec(ctx context.Context, query string, args ...any) (DbWrapperResult, error)
+}