refactor: adjust configuration parameters for extractors and transformers; optimize batch processing settings

config.yaml

@@ -70,11 +70,11 @@ jobs:
         - source: DATA
           target: FILE_URL
           mode: REFERENCE_ONLY
-    batches_per_partition: 1000
-    max_extractors: 8
+    batches_per_partition: 20
+    max_extractors: 32
     extractor_batch_size: 1
-    extractor_queue_size: 64
-    max_transformers: 16
+    extractor_queue_size: 100
+    max_transformers: 48
     transformer_batch_size: 500
     transformer_queue_size: 8
     max_loaders: 4

scripts/mssql-copy-in/main.go

@@ -13,8 +13,8 @@ import (
 
 const (
 	// totalRows int = 1_000_000
-	totalRows int = 5000
-	chunkSize int = 500
+	totalRows int = 1000
+	chunkSize int = 200
 	queueSize int = 4
 )