max_parallel_workers: 4
source_db_type: sqlserver
target_db_type: postgres

defaults:
  batches_per_partition: 4
  max_extractors: 2
  extractor_batch_size: 5000
  extractor_queue_size: 8
  max_transformers: 2
  transformer_batch_size: 12500
  transformer_queue_size: 8
  max_loaders: 4
  loader_batch_size: 25000
  partition_calculation_strategy: EXACT # EXACT | ESTIMATION
  truncate_target: true
  truncate_method: TRUNCATE # TRUNCATE | DELETE
  retry:
    attempts: 3
    base_delay_ms: 500
    max_delay_ms: 10000
    max_jitter_ms: 500
    max_failed_partitions: 5
    max_failed_batches_load: 5

jobs:
  - name: demo_users
    enabled: true
    source:
      schema: demo
      table: users
      primary_key: id
    target:
      schema: demo
      table: users
      pre_sql: 
        - 'SELECT 1'
    range:
      min: 1000000
      max: 2000000
      is_min_inclusive: false
      is_max_inclusive: true

  - name: analytics_events
    enabled: true
    source:
      schema: analytics
      table: events
      primary_key: ID_events
      from_json:
        - column: $node_id*
          field: id
    target:
      schema: analytics
      table: events
      pre_sql: 
        - 'SELECT 1'
      post_sql: 
        - "SELECT 1"

  - name: analytics_audit_log
    source:
      schema: storage
      table: attachments
      primary_key: id
    target:
      schema: storage
      table: attachments
    to_storage:
      columns:
        - source: DATA
          target: FILE_URL
          mode: REFERENCE_ONLY
          prefix: storage/attachments
    batches_per_partition: 20
    max_extractors: 32
    extractor_batch_size: 1
    extractor_queue_size: 100
    max_transformers: 48
    transformer_batch_size: 500
    transformer_queue_size: 8
    max_loaders: 4
    loader_batch_size: 500
    retry:
      attempts: 5
      base_delay_ms: 1000
      max_delay_ms: 15000
      max_jitter_ms: 500