refactor: rename Batch to Partition across pipeline

Rename internal types and channels from Batch to Partition for consistency with the data model.
This commit is contained in:
2026-04-11 18:00:00 -05:00
parent 1f8689cdfc
commit 7f3d2b8cc4
28 changed files with 1250 additions and 781 deletions

View File

@@ -1,11 +1,13 @@
max_parallel_workers: 2
max_parallel_workers: 4
source_db_type: sqlserver
target_db_type: postgres
defaults:
max_extractors: 4
max_loaders: 8
max_extractors: 2
max_loaders: 4
queue_size: 8
chunk_size: 50000
chunks_per_batch: 10
chunk_size: 25000
chunks_per_batch: 8
truncate_target: true
truncate_method: TRUNCATE # TRUNCATE | DELETE
retry:
@@ -21,19 +23,6 @@ jobs:
target:
schema: demo
table: users
max_extractors: 2 # overrides default config
max_loaders: 4 # overrides default config
queue_size: 4 # overrides default config
chunk_size: 25000 # overrides default config
chunks_per_batch: 8 # overrides default config
truncate_target: false # overrides default config
truncate_method: DELETE # overrides default config
retry:
attempts: 5 # overrides default config
pre_sql:
- "SELECT 1"
post_sql:
- "SELECT 2"
- name: analytics_events
enabled: true