max_parallel_workers: 4
source_db_type: sqlserver
target_db_type: postgres

defaults:
  batches_per_partition: 4
  max_extractors: 2
  extractor_batch_size: 5000
  extractor_queue_size: 8
  max_transformers: 2
  transformer_batch_size: 12500
  transformer_queue_size: 8
  max_loaders: 4
  loader_batch_size: 25000
  partition_calculation_strategy: EXACT # EXACT | ESTIMATION
  truncate_target: true
  truncate_method: TRUNCATE # TRUNCATE | DELETE
  retry:
    attempts: 3
    base_delay_ms: 500
    max_delay_ms: 10000
    max_jitter_ms: 500
    max_failed_partitions: 5
    max_failed_batches_load: 5

jobs:
  - name: demo_users
    enabled: true
    source:
      schema: demo
      table: users
      primary_key: id
    target:
      schema: demo
      table: users

  # - name: analytics_events
  #   enabled: true
  #   source:
  #     schema: analytics
  #     table: events
  #     primary_key: id
  #     from_json:
  #       - column: payload
  #         field: id
  #   target:
  #     schema: analytics
  #     table: events

  # - name: storage_attachments
  #   source:
  #     schema: storage
  #     table: attachments
  #     primary_key: id
  #   target:
  #     schema: storage
  #     table: attachments
  #   to_storage:
  #     columns:
  #       - source: DATA
  #         target: FILE_URL
  #         mode: REFERENCE_ONLY
  #         prefix: storage/attachments
  #   batches_per_partition: 20
  #   max_extractors: 32
  #   extractor_batch_size: 1
  #   extractor_queue_size: 100
  #   max_transformers: 48
  #   transformer_batch_size: 500
  #   transformer_queue_size: 8
  #   max_loaders: 4
  #   loader_batch_size: 500
  #   retry:
  #     attempts: 5
  #     base_delay_ms: 1000
  #     max_delay_ms: 15000
  #     max_jitter_ms: 500

  # - name: analytics_audit_log
  #   source:
  #     schema: analytics
  #     table: audit_log
  #     primary_key: id
  #   target:
  #     schema: analytics
  #     table: audit_log
  #     pre_sql:
  #       - "DROP INDEX IF EXISTS audit_log_created_at_idx"
  #       - "DROP INDEX IF EXISTS audit_log_user_id_idx"
  #     post_sql:
  #       - "CREATE INDEX audit_log_created_at_idx ON analytics.audit_log (created_at)"
  #       - "CREATE INDEX audit_log_user_id_idx ON analytics.audit_log (user_id)"
  #       - "ANALYZE analytics.audit_log"
  #   # Partition slicing: only migrate the slice [2024-01-01, 2024-12-31]
  #   # in this re-run. is_min_inclusive=false resumes AFTER any failed boundary.
  #   range:
  #     min: 1704067200000   # 2024-01-01 UTC, as epoch millis
  #     max: 1735689599999   # 2024-12-31 UTC, as epoch millis
  #     is_min_inclusive: false
  #     is_max_inclusive: true
  #   partition_calculation_strategy: ESTIMATION  # faster than EXACT on large tables
  #   truncate_method: DELETE                    # use DELETE instead of TRUNCATE
  #   batches_per_partition: 8
  #   retry:
  #     attempts: 5
  #     max_failed_partitions: 3
  #     max_failed_batches_load: 10