kylesoda/go-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: adjust configuration parameters for extractors and transformers; optimize batch processing settings

Browse Source
This commit is contained in:
Kylesoda
2026-05-09 02:07:09 -05:00
parent 6fa9b21b1c
commit 0c59d06af6
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
config.yaml
View File

@@ -70,11 +70,11 @@ jobs:
- source: DATA - source: DATA
target: FILE_URL target: FILE_URL
mode: REFERENCE_ONLY mode: REFERENCE_ONLY
batches_per_partition: 1000 batches_per_partition: 20
max_extractors: 8 max_extractors: 32
extractor_batch_size: 1 extractor_batch_size: 1
extractor_queue_size: 64 extractor_queue_size: 100
max_transformers: 16 max_transformers: 48
transformer_batch_size: 500 transformer_batch_size: 500
transformer_queue_size: 8 transformer_queue_size: 8
max_loaders: 4 max_loaders: 4

4
scripts/mssql-copy-in/main.go
View File

@@ -13,8 +13,8 @@ import (
const ( const (
// totalRows int = 1_000_000 // totalRows int = 1_000_000
totalRows int = 5000 totalRows int = 1000
chunkSize int = 500 chunkSize int = 200
queueSize int = 4 queueSize int = 4
) )