diff --git a/config.yaml b/config.yaml index f0afca6..aa9ac6f 100644 --- a/config.yaml +++ b/config.yaml @@ -70,11 +70,11 @@ jobs: - source: DATA target: FILE_URL mode: REFERENCE_ONLY - batches_per_partition: 1000 - max_extractors: 8 + batches_per_partition: 20 + max_extractors: 32 extractor_batch_size: 1 - extractor_queue_size: 64 - max_transformers: 16 + extractor_queue_size: 100 + max_transformers: 48 transformer_batch_size: 500 transformer_queue_size: 8 max_loaders: 4 diff --git a/scripts/mssql-copy-in/main.go b/scripts/mssql-copy-in/main.go index 43a8d67..ea251b5 100644 --- a/scripts/mssql-copy-in/main.go +++ b/scripts/mssql-copy-in/main.go @@ -13,8 +13,8 @@ import ( const ( // totalRows int = 1_000_000 - totalRows int = 5000 - chunkSize int = 500 + totalRows int = 1000 + chunkSize int = 200 queueSize int = 4 )