feat: enhance logging and performance metrics for migration processes

cmd/go_migrate/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"time"
+
 	log "github.com/sirupsen/logrus"
 )
 
@@ -20,15 +22,16 @@ var migrationJobs []MigrationJob = []MigrationJob{
 
 const (
 	NumExtractors int = 1
-	NumLoaders    int = 2
-	ChunkSize     int = 50000
+	NumLoaders    int = 4
+	ChunkSize     int = 100000
 	QueueSize     int = 10
 )
 
 func main() {
 	configureLog()
-	log.Info("Starting migration...")
-	// log.Debugf("Migration jobs: %+v", migrationJobs)
+	startTime := time.Now()
+	log.Info("=== Starting migration ===")
+	log.Infof("Number of loaders: %d, Chunk size: %d", NumLoaders, ChunkSize)
 
 	sourceDb, targetDb, connError := connectToDatabases()
 	if connError != nil {
@@ -39,9 +42,11 @@ func main() {
 	defer targetDb.Close()
 
 	for _, job := range migrationJobs {
-		log.Infof("Processing job: %+v", job)
+		log.Infof(">>> Processing job: %s.%s <<<", job.Schema, job.Table)
 		processMigrationJob(sourceDb, targetDb, job)
 	}
 
-	log.Info("Migration completed successfully!")
+	totalDuration := time.Since(startTime)
+	log.Infof("=== Migration completed successfully! ===")
+	log.Infof("Total migration time: %v", totalDuration)
 }