package loaders import ( "context" "errors" "fmt" "sync" "sync/atomic" "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/config" "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/custom_errors" "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/etl" "git.ksdemosapps.com/kylesoda/go-migrate/internal/app/models" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" ) type PostgresLoader struct { db *pgxpool.Pool } func NewPostgresLoader(pool *pgxpool.Pool) etl.Loader { return &PostgresLoader{db: pool} } func mapSlice[T any, V any](input []T, mapper func(T) V) []V { result := make([]V, len(input)) for i, v := range input { result[i] = mapper(v) } return result } func (postgresLd *PostgresLoader) ProcessChunk( ctx context.Context, tableInfo config.TargetTableInfo, colNames []string, chunk models.Batch, ) (int, error) { tableId := pgx.Identifier{tableInfo.Schema, tableInfo.Table} _, err := postgresLd.db.CopyFrom( ctx, tableId, colNames, pgx.CopyFromRows(chunk.Data), ) if err != nil { var pgErr *pgconn.PgError if errors.As(err, &pgErr) { if pgErr.Code == "23505" { return 0, &custom_errors.JobError{ ShouldCancelJob: true, Msg: fmt.Sprintf("Fatal error in table %s", tableId.Sanitize()), Prev: err, } } } return 0, &custom_errors.LoaderError{Batch: chunk, Msg: err.Error()} } return len(chunk.Data), nil } func (postgresLd *PostgresLoader) Exec( ctx context.Context, tableInfo config.TargetTableInfo, columns []models.ColumnType, chChunksIn <-chan models.Batch, chErrorsOut chan<- custom_errors.LoaderError, chJobErrorsOut chan<- custom_errors.JobError, wgActiveChunks *sync.WaitGroup, rowsLoaded *int64, ) { colNames := mapSlice(columns, func(col models.ColumnType) string { return col.Name() }) for { if ctx.Err() != nil { return } select { case <-ctx.Done(): return case chunk, ok := <-chChunksIn: if !ok { return } processedRows, err := postgresLd.ProcessChunk(ctx, tableInfo, colNames, chunk) if err != nil { var ldError *custom_errors.LoaderError if errors.As(err, &ldError) { select { case <-ctx.Done(): return case chErrorsOut <- *ldError: } } var jobError *custom_errors.JobError if errors.As(err, &jobError) { select { case <-ctx.Done(): return case chJobErrorsOut <- *jobError: } } return } wgActiveChunks.Done() atomic.AddInt64(rowsLoaded, int64(processedRows)) } } }