Kylesoda
1f8689cdfc
Add a YAML config schema for migration jobs (source/target schema, table, primary key, options) and parse it at startup.
47 lines
1.0 KiB
YAML
47 lines
1.0 KiB
YAML
max_parallel_workers: 2
|
|
|
|
defaults:
|
|
max_extractors: 4
|
|
max_loaders: 8
|
|
queue_size: 8
|
|
chunk_size: 50000
|
|
chunks_per_batch: 10
|
|
truncate_target: true
|
|
truncate_method: TRUNCATE # TRUNCATE | DELETE
|
|
retry:
|
|
attempts: 3
|
|
|
|
jobs:
|
|
- name: demo_users
|
|
enabled: true
|
|
source:
|
|
schema: demo
|
|
table: users
|
|
primary_key: id
|
|
target:
|
|
schema: demo
|
|
table: users
|
|
max_extractors: 2 # overrides default config
|
|
max_loaders: 4 # overrides default config
|
|
queue_size: 4 # overrides default config
|
|
chunk_size: 25000 # overrides default config
|
|
chunks_per_batch: 8 # overrides default config
|
|
truncate_target: false # overrides default config
|
|
truncate_method: DELETE # overrides default config
|
|
retry:
|
|
attempts: 5 # overrides default config
|
|
pre_sql:
|
|
- "SELECT 1"
|
|
post_sql:
|
|
- "SELECT 2"
|
|
|
|
- name: analytics_events
|
|
enabled: true
|
|
source:
|
|
schema: analytics
|
|
table: events
|
|
primary_key: ID_events
|
|
target:
|
|
schema: analytics
|
|
table: events
|