bfe335c933
- Deleted `seed-manzana.go`, `site-holder-attach.go`, `types.go`, `utils.go`, `main.go` files from both `mssql-info-test` and `mssql-test` directories. - Removed `pg-info-test/main.go` and `test-etl/main.go` files. - Eliminated `wkb-to-ewkb/main.go` for unused WKB to EWKB conversion functionality. - Cleaned up associated functions and types that are no longer needed.
108 lines
2.9 KiB
YAML
108 lines
2.9 KiB
YAML
max_parallel_workers: 4
|
|
source_db_type: sqlserver
|
|
target_db_type: postgres
|
|
|
|
defaults:
|
|
batches_per_partition: 4
|
|
max_extractors: 2
|
|
extractor_batch_size: 5000
|
|
extractor_queue_size: 8
|
|
max_transformers: 2
|
|
transformer_batch_size: 12500
|
|
transformer_queue_size: 8
|
|
max_loaders: 4
|
|
loader_batch_size: 25000
|
|
partition_calculation_strategy: EXACT # EXACT | ESTIMATION
|
|
truncate_target: true
|
|
truncate_method: TRUNCATE # TRUNCATE | DELETE
|
|
retry:
|
|
attempts: 3
|
|
base_delay_ms: 500
|
|
max_delay_ms: 10000
|
|
max_jitter_ms: 500
|
|
max_failed_partitions: 5
|
|
max_failed_batches_load: 5
|
|
|
|
jobs:
|
|
- name: demo_users
|
|
enabled: true
|
|
source:
|
|
schema: demo
|
|
table: users
|
|
primary_key: id
|
|
target:
|
|
schema: demo
|
|
table: users
|
|
|
|
# - name: analytics_events
|
|
# enabled: true
|
|
# source:
|
|
# schema: analytics
|
|
# table: events
|
|
# primary_key: id
|
|
# from_json:
|
|
# - column: payload
|
|
# field: id
|
|
# target:
|
|
# schema: analytics
|
|
# table: events
|
|
|
|
# - name: storage_attachments
|
|
# source:
|
|
# schema: storage
|
|
# table: attachments
|
|
# primary_key: id
|
|
# target:
|
|
# schema: storage
|
|
# table: attachments
|
|
# to_storage:
|
|
# columns:
|
|
# - source: DATA
|
|
# target: FILE_URL
|
|
# mode: REFERENCE_ONLY
|
|
# prefix: storage/attachments
|
|
# batches_per_partition: 20
|
|
# max_extractors: 32
|
|
# extractor_batch_size: 1
|
|
# extractor_queue_size: 100
|
|
# max_transformers: 48
|
|
# transformer_batch_size: 500
|
|
# transformer_queue_size: 8
|
|
# max_loaders: 4
|
|
# loader_batch_size: 500
|
|
# retry:
|
|
# attempts: 5
|
|
# base_delay_ms: 1000
|
|
# max_delay_ms: 15000
|
|
# max_jitter_ms: 500
|
|
|
|
# - name: analytics_audit_log
|
|
# source:
|
|
# schema: analytics
|
|
# table: audit_log
|
|
# primary_key: id
|
|
# target:
|
|
# schema: analytics
|
|
# table: audit_log
|
|
# pre_sql:
|
|
# - "DROP INDEX IF EXISTS audit_log_created_at_idx"
|
|
# - "DROP INDEX IF EXISTS audit_log_user_id_idx"
|
|
# post_sql:
|
|
# - "CREATE INDEX audit_log_created_at_idx ON analytics.audit_log (created_at)"
|
|
# - "CREATE INDEX audit_log_user_id_idx ON analytics.audit_log (user_id)"
|
|
# - "ANALYZE analytics.audit_log"
|
|
# # Partition slicing: only migrate the slice [2024-01-01, 2024-12-31]
|
|
# # in this re-run. is_min_inclusive=false resumes AFTER any failed boundary.
|
|
# range:
|
|
# min: 1704067200000 # 2024-01-01 UTC, as epoch millis
|
|
# max: 1735689599999 # 2024-12-31 UTC, as epoch millis
|
|
# is_min_inclusive: false
|
|
# is_max_inclusive: true
|
|
# partition_calculation_strategy: ESTIMATION # faster than EXACT on large tables
|
|
# truncate_method: DELETE # use DELETE instead of TRUNCATE
|
|
# batches_per_partition: 8
|
|
# retry:
|
|
# attempts: 5
|
|
# max_failed_partitions: 3
|
|
# max_failed_batches_load: 10
|