package main import ( "context" "fmt" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" log "github.com/sirupsen/logrus" ) func fakeLoader(job MigrationJob, columns []ColumnType, in <-chan [][]any) { for rows := range in { log.Debugf("Chunk received, loading data into...") for i, rowValues := range rows { if i%100 == 0 { logSampleRow(job, columns, rowValues, fmt.Sprintf("row %d", i)) } } } } func loadRowsPostgres(ctx context.Context, job MigrationJob, columns []ColumnType, db *pgxpool.Pool, in <-chan []UnknownRowValues) error { chunkCount := 0 totalRowsLoaded := 0 for rows := range in { chunkStartTime := time.Now() identifier := pgx.Identifier{job.Schema, job.Table} colNames := Map(columns, func(col ColumnType) string { return col.name }) copyStartTime := time.Now() _, err := db.CopyFrom( ctx, identifier, colNames, pgx.CopyFromRows(rows), ) if err != nil { return err } chunkCount++ totalRowsLoaded += len(rows) copyDuration := time.Since(copyStartTime) chunkDuration := time.Since(chunkStartTime) rowsPerSec := float64(len(rows)) / chunkDuration.Seconds() log.Infof("Loaded chunk #%d: %d rows in %v (copy: %v, %.0f rows/sec) - Total: %d rows", chunkCount, len(rows), chunkDuration, copyDuration, rowsPerSec, totalRowsLoaded) } return nil } func Map[T any, V any](input []T, mapper func(T) V) []V { result := make([]V, len(input)) for i, v := range input { result[i] = mapper(v) } return result }