refactor: consolidate error handling and retry configuration

Centralize the error channel size, retry policy and max-failed batches/partition limits in the job config.
This commit is contained in:
2026-05-05 16:00:00 -05:00
parent 70ce71f59b
commit bf15979c00
15 changed files with 356 additions and 363 deletions

View File

@@ -3,15 +3,19 @@ source_db_type: sqlserver
target_db_type: postgres
defaults:
max_extractors: 2
max_loaders: 4
queue_size: 8
batch_size: 25000
batches_per_partition: 8
max_extractors: 2
extractor_batch_size: 25000
extractor_queue_size: 8
max_transformers: 2
transformer_batch_size: 25000
transformer_queue_size: 8
max_loaders: 4
loader_batch_size: 25000
truncate_target: true
truncate_method: TRUNCATE # TRUNCATE | DELETE
max_partition_errrors: 5
max_chunk_errors: 5
max_extractor_batch_errors: 5
retry:
attempts: 3
base_delay_ms: 500