feat: implement batch processing for MSSQL with improved structure and logging

2026-04-08 17:35:12 -05:00
parent 51d83661a4
commit bc6f9a6a70
8 changed files with 343 additions and 168 deletions
--- a/cmd/go_migrate/main.go
+++ b/cmd/go_migrate/main.go
@@ -26,10 +26,12 @@ var migrationJobs []MigrationJob = []MigrationJob{
 }

 const (
-	NumExtractors   int = 4
-	NumLoaders      int = 8
-	ChunkSize       int = 25000
-	QueueSize       int = 8
+	NumExtractors  int   = 4
+	NumLoaders     int   = 8
+	ChunkSize      int   = 25000
+	QueueSize      int   = 8
+	ChunksPerBatch int   = 16
+	RowsPerBatch   int64 = int64(ChunkSize * ChunksPerBatch)
 )

 func main() {