70 lines
1.5 KiB
Go
70 lines
1.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
func fakeLoader(job MigrationJob, columns []ColumnType, in <-chan [][]any) {
|
|
for rows := range in {
|
|
log.Debugf("Chunk received, loading data into...")
|
|
|
|
for i, rowValues := range rows {
|
|
if i%100 == 0 {
|
|
logSampleRow(job, columns, rowValues, fmt.Sprintf("row %d", i))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func loadRowsPostgres(ctx context.Context, job MigrationJob, columns []ColumnType, db *pgxpool.Pool, in <-chan []UnknownRowValues) error {
|
|
chunkCount := 0
|
|
totalRowsLoaded := 0
|
|
|
|
for rows := range in {
|
|
chunkStartTime := time.Now()
|
|
identifier := pgx.Identifier{job.Schema, job.Table}
|
|
colNames := Map(columns, func(col ColumnType) string {
|
|
return col.name
|
|
})
|
|
|
|
copyStartTime := time.Now()
|
|
_, err := db.CopyFrom(
|
|
ctx,
|
|
identifier,
|
|
colNames,
|
|
pgx.CopyFromRows(rows),
|
|
)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
chunkCount++
|
|
totalRowsLoaded += len(rows)
|
|
copyDuration := time.Since(copyStartTime)
|
|
chunkDuration := time.Since(chunkStartTime)
|
|
rowsPerSec := float64(len(rows)) / chunkDuration.Seconds()
|
|
|
|
log.Infof("Loaded chunk #%d: %d rows in %v (copy: %v, %.0f rows/sec) - Total: %d rows",
|
|
chunkCount, len(rows), chunkDuration, copyDuration, rowsPerSec, totalRowsLoaded)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func Map[T any, V any](input []T, mapper func(T) V) []V {
|
|
result := make([]V, len(input))
|
|
|
|
for i, v := range input {
|
|
result[i] = mapper(v)
|
|
}
|
|
|
|
return result
|
|
}
|