From 0c59d06af6cb31d04b7fd347f41befb2e985a5b0 Mon Sep 17 00:00:00 2001
From: Kylesoda <249518290+kylesoda@users.noreply.github.com>
Date: Sat, 9 May 2026 02:07:09 -0500
Subject: [PATCH] refactor: adjust configuration parameters for extractors and
 transformers; optimize batch processing settings

---
 config.yaml                   | 8 ++++----
 scripts/mssql-copy-in/main.go | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/config.yaml b/config.yaml
index f0afca6..aa9ac6f 100644
--- a/config.yaml
+++ b/config.yaml
@@ -70,11 +70,11 @@ jobs:
         - source: DATA
           target: FILE_URL
           mode: REFERENCE_ONLY
-    batches_per_partition: 1000
-    max_extractors: 8
+    batches_per_partition: 20
+    max_extractors: 32
     extractor_batch_size: 1
-    extractor_queue_size: 64
-    max_transformers: 16
+    extractor_queue_size: 100
+    max_transformers: 48
     transformer_batch_size: 500
     transformer_queue_size: 8
     max_loaders: 4
diff --git a/scripts/mssql-copy-in/main.go b/scripts/mssql-copy-in/main.go
index 43a8d67..ea251b5 100644
--- a/scripts/mssql-copy-in/main.go
+++ b/scripts/mssql-copy-in/main.go
@@ -13,8 +13,8 @@ import (
 
 const (
 	// totalRows int = 1_000_000
-	totalRows int = 5000
-	chunkSize int = 500
+	totalRows int = 1000
+	chunkSize int = 200
 	queueSize int = 4
 )