73 lines
1.6 KiB
Go
73 lines
1.6 KiB
Go
package main
|
|
|
|
import (
|
|
"time"
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
func transformRowsMssql(
|
|
columns []ColumnType,
|
|
chChunksIn <-chan Chunk,
|
|
chChunksOut chan<- Chunk,
|
|
chJobErrorsOut chan<- JobError,
|
|
) {
|
|
chunkCount := 0
|
|
totalRowsTransformed := 0
|
|
|
|
for chunk := range chChunksIn {
|
|
chunkStartTime := time.Now()
|
|
log.Debugf("Chunk received, transforming %d rows...", len(chunk.Data))
|
|
|
|
for _, rowValues := range chunk.Data {
|
|
for i, col := range columns {
|
|
value := rowValues[i]
|
|
|
|
switch col.SystemType() {
|
|
case "uniqueidentifier":
|
|
if b, ok := value.([]byte); ok {
|
|
pgUuid, err := mssqlUuidToBigEndian(b)
|
|
if err != nil {
|
|
jobError := JobError{
|
|
ShouldCancelJob: true,
|
|
Prev: err,
|
|
}
|
|
chJobErrorsOut <- jobError
|
|
return
|
|
}
|
|
rowValues[i] = pgUuid
|
|
}
|
|
|
|
case "geometry", "geography":
|
|
if b, ok := value.([]byte); ok {
|
|
ewkb, err := wkbToEwkbWithSrid(b, 4326)
|
|
if err != nil {
|
|
jobError := JobError{
|
|
ShouldCancelJob: true,
|
|
Prev: err,
|
|
}
|
|
chJobErrorsOut <- jobError
|
|
return
|
|
}
|
|
rowValues[i] = ewkb
|
|
}
|
|
|
|
case "datetime", "datetime2":
|
|
if t, ok := value.(time.Time); ok {
|
|
rowValues[i] = ensureUTC(t)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
chunkCount++
|
|
totalRowsTransformed += len(chunk.Data)
|
|
chunkDuration := time.Since(chunkStartTime)
|
|
rowsPerSec := float64(len(chunk.Data)) / chunkDuration.Seconds()
|
|
log.Infof("Transformed chunk: %d rows in %v (%.0f rows/sec) - Total: %d rows",
|
|
len(chunk.Data), chunkDuration, rowsPerSec, totalRowsTransformed)
|
|
|
|
chChunksOut <- chunk
|
|
}
|
|
}
|