feat: add MSSQL table analyzer and integrate partition range generation for improved data migration
This commit is contained in:
40
internal/app/etl/table_analyzers/main.go
Normal file
40
internal/app/etl/table_analyzers/main.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package table_analyzers
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl"
|
||||
"git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func PartitionRangeGenerator(
|
||||
ctx context.Context,
|
||||
tableAnalyzer etl.TableAnalyzer,
|
||||
tableInfo config.TableInfo,
|
||||
partitionColumn string,
|
||||
rowsPerPartition int64,
|
||||
) ([]models.Partition, error) {
|
||||
rowsCount, err := tableAnalyzer.EstimateTotalRows(ctx, tableInfo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if rowsCount <= rowsPerPartition {
|
||||
return []models.Partition{{
|
||||
Id: uuid.New(),
|
||||
ShouldUseRange: false,
|
||||
RetryCounter: 0,
|
||||
}}, nil
|
||||
|
||||
}
|
||||
|
||||
partitionsCount := rowsCount / rowsPerPartition
|
||||
partitions, err := tableAnalyzer.CalculatePartitionRanges(ctx, tableInfo, partitionColumn, partitionsCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return partitions, nil
|
||||
}
|
||||
Reference in New Issue
Block a user