Kylesoda
933a7572d5
Stream transformer output to an Azure blob container when a job declares ToStorage, with per-job prefix and credential configuration.
66 lines
1.3 KiB
YAML
66 lines
1.3 KiB
YAML
max_parallel_workers: 4
|
|
source_db_type: sqlserver
|
|
target_db_type: postgres
|
|
|
|
defaults:
|
|
max_extractors: 2
|
|
max_loaders: 4
|
|
queue_size: 8
|
|
batch_size: 25000
|
|
batches_per_partition: 8
|
|
truncate_target: true
|
|
truncate_method: TRUNCATE # TRUNCATE | DELETE
|
|
max_partition_errrors: 5
|
|
max_chunk_errors: 5
|
|
retry:
|
|
attempts: 3
|
|
base_delay_ms: 500
|
|
max_delay_ms: 10000
|
|
max_jitter_ms: 500
|
|
|
|
jobs:
|
|
- name: demo_users
|
|
enabled: true
|
|
source:
|
|
schema: demo
|
|
table: users
|
|
primary_key: id
|
|
target:
|
|
schema: demo
|
|
table: users
|
|
pre_sql:
|
|
- 'SELECT 1'
|
|
range:
|
|
min: 1000000
|
|
max: 2000000
|
|
is_min_inclusive: false
|
|
is_max_inclusive: true
|
|
|
|
- name: analytics_events
|
|
enabled: true
|
|
source:
|
|
schema: analytics
|
|
table: events
|
|
primary_key: ID_events
|
|
target:
|
|
schema: analytics
|
|
table: events
|
|
pre_sql:
|
|
- 'SELECT 1'
|
|
post_sql:
|
|
- "SELECT 1"
|
|
|
|
- name: storage_attachments
|
|
source:
|
|
schema: analytics
|
|
table: attachments
|
|
primary_key: id
|
|
target:
|
|
schema: analytics
|
|
table: attachments
|
|
to_storage:
|
|
columns:
|
|
- source: DATA
|
|
target: FILE_URL
|
|
mode: REFERENCE_ONLY # REFERENCE_ONLY | DUPLICATE_WITH_REF
|