feat: add configuration parsing and job definitions
Add a YAML config schema for migration jobs (source/target schema, table, primary key, options) and parse it at startup.
This commit is contained in:
46
config.yaml
Normal file
46
config.yaml
Normal file
@@ -0,0 +1,46 @@
|
||||
max_parallel_workers: 2
|
||||
|
||||
defaults:
|
||||
max_extractors: 4
|
||||
max_loaders: 8
|
||||
queue_size: 8
|
||||
chunk_size: 50000
|
||||
chunks_per_batch: 10
|
||||
truncate_target: true
|
||||
truncate_method: TRUNCATE # TRUNCATE | DELETE
|
||||
retry:
|
||||
attempts: 3
|
||||
|
||||
jobs:
|
||||
- name: demo_users
|
||||
enabled: true
|
||||
source:
|
||||
schema: demo
|
||||
table: users
|
||||
primary_key: id
|
||||
target:
|
||||
schema: demo
|
||||
table: users
|
||||
max_extractors: 2 # overrides default config
|
||||
max_loaders: 4 # overrides default config
|
||||
queue_size: 4 # overrides default config
|
||||
chunk_size: 25000 # overrides default config
|
||||
chunks_per_batch: 8 # overrides default config
|
||||
truncate_target: false # overrides default config
|
||||
truncate_method: DELETE # overrides default config
|
||||
retry:
|
||||
attempts: 5 # overrides default config
|
||||
pre_sql:
|
||||
- "SELECT 1"
|
||||
post_sql:
|
||||
- "SELECT 2"
|
||||
|
||||
- name: analytics_events
|
||||
enabled: true
|
||||
source:
|
||||
schema: analytics
|
||||
table: events
|
||||
primary_key: ID_events
|
||||
target:
|
||||
schema: analytics
|
||||
table: events
|
||||
Reference in New Issue
Block a user