Kylesoda
74abf12dcf
Introduce a retry policy with exponential backoff and jitter for extractor and loader errors, with configurable max attempts and delay caps.
39 lines
708 B
YAML
39 lines
708 B
YAML
max_parallel_workers: 4
|
|
source_db_type: sqlserver
|
|
target_db_type: postgres
|
|
|
|
defaults:
|
|
max_extractors: 2
|
|
max_loaders: 4
|
|
queue_size: 8
|
|
batch_size: 25000
|
|
batches_per_partition: 8
|
|
truncate_target: true
|
|
truncate_method: TRUNCATE # TRUNCATE | DELETE
|
|
retry:
|
|
attempts: 3
|
|
base_delay_ms: 500
|
|
max_delay_ms: 10000
|
|
max_jitter_ms: 500
|
|
|
|
jobs:
|
|
- name: demo_users
|
|
enabled: true
|
|
source:
|
|
schema: demo
|
|
table: users
|
|
primary_key: id
|
|
target:
|
|
schema: demo
|
|
table: users
|
|
|
|
- name: analytics_events
|
|
enabled: true
|
|
source:
|
|
schema: analytics
|
|
table: events
|
|
primary_key: ID_events
|
|
target:
|
|
schema: analytics
|
|
table: events
|