feat: add configuration parsing and job definitions

Add a YAML config schema for migration jobs (source/target schema, table, primary key, options) and parse it at startup.
This commit is contained in:
2026-04-10 14:00:00 -05:00
parent 4f181269a4
commit 1f8689cdfc
16 changed files with 1226 additions and 30 deletions

46
config.yaml Normal file
View File

@@ -0,0 +1,46 @@
max_parallel_workers: 2
defaults:
max_extractors: 4
max_loaders: 8
queue_size: 8
chunk_size: 50000
chunks_per_batch: 10
truncate_target: true
truncate_method: TRUNCATE # TRUNCATE | DELETE
retry:
attempts: 3
jobs:
- name: demo_users
enabled: true
source:
schema: demo
table: users
primary_key: id
target:
schema: demo
table: users
max_extractors: 2 # overrides default config
max_loaders: 4 # overrides default config
queue_size: 4 # overrides default config
chunk_size: 25000 # overrides default config
chunks_per_batch: 8 # overrides default config
truncate_target: false # overrides default config
truncate_method: DELETE # overrides default config
retry:
attempts: 5 # overrides default config
pre_sql:
- "SELECT 1"
post_sql:
- "SELECT 2"
- name: analytics_events
enabled: true
source:
schema: analytics
table: events
primary_key: ID_events
target:
schema: analytics
table: events