Skip to content

Instantly share code, notes, and snippets.

@amustafa
Last active March 24, 2021 13:58
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save amustafa/27ffc4c8debaeb05dbdf9608854bf670 to your computer and use it in GitHub Desktop.
Save amustafa/27ffc4c8debaeb05dbdf9608854bf670 to your computer and use it in GitHub Desktop.
This is an example script that causes Apache Arrow ipc.Writer in the Go library to panic. (https://pkg.go.dev/github.com/apache/arrow/go/arrow@v0.0.0-20210127213842-4a9885e30a98/ipc#Writer)
/*
panic: arrow/array: index out of range
goroutine 1 [running]:
github.com/apache/arrow/go/arrow/array.NewSliceData(0xc000182a20, 0x8, 0x1, 0x8904008)
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/array/data.go:149 +0x1c6
github.com/apache/arrow/go/arrow/ipc.(*recordEncoder).visit(0xc0001afa30, 0xc0001afa00, 0x1169180, 0xc000186740, 0x0, 0x0)
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/ipc/writer.go:273 +0x15c5
github.com/apache/arrow/go/arrow/ipc.(*recordEncoder).Encode(0xc0001afa30, 0xc0001afa00, 0x11696c0, 0xc00018e2d0, 0x10, 0x1773fd8)
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/ipc/writer.go:178 +0x285
github.com/apache/arrow/go/arrow/ipc.(*Writer).Write(0xc000186b00, 0x11696c0, 0xc00018e2d0, 0x0, 0x0)
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/ipc/writer.go:130 +0x1a8
main.encodeDataArrow(0x11696c0, 0xc00018e2d0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users//workspace/record_serialize.go:46 +0x14c
main.encodeDecode(0xc0001869c0, 0x3, 0x4)
/Users//workspace/record_serialize.go:69 +0x219
main.main()
/Users//workspace/record_serialize.go:127 +0x793
exit status 2
*/
package main
import (
"bytes"
"errors"
"fmt"
"math/rand"
"strconv"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
arrowArray "github.com/apache/arrow/go/arrow/array"
arrowIpc "github.com/apache/arrow/go/arrow/ipc"
"github.com/apache/arrow/go/arrow/memory"
)
func encodeDataArrow(rec arrowArray.Record) ([]byte, error) {
rec.Retain()
defer rec.Release()
var ipcOutput bytes.Buffer
w := arrowIpc.NewWriter(&ipcOutput, arrowIpc.WithSchema(rec.Schema()))
defer w.Close()
if err := w.Write(rec); err != nil {
return nil, err
}
b := ipcOutput.Bytes()
return b, nil
}
func decodeArrow(data []byte) (arrowArray.Record, error) {
buf := bytes.NewBuffer(data)
reader, err := arrowIpc.NewReader(buf)
if err != nil {
return nil, err
}
for reader.Next() {
rec := reader.Record()
rec.Retain()
return rec, nil
}
return nil, errors.New("didn't get any records")
}
func encodeDecode(data []arrowArray.Record) error {
for _, record := range data {
bytes, err := encodeDataArrow(record)
if err != nil {
return err
}
deserializedData, err := decodeArrow(bytes)
if err != nil {
return err
}
fmt.Println(deserializedData)
}
return nil
}
func main() {
size := 3
schema := arrow.NewSchema(
[]arrow.Field{
arrow.Field{Name: "idx", Type: arrow.PrimitiveTypes.Int64},
arrow.Field{Name: "A", Type: arrow.PrimitiveTypes.Int64},
arrow.Field{Name: "B", Type: arrow.PrimitiveTypes.Int64},
arrow.Field{Name: "C", Type: arrow.BinaryTypes.String},
},
nil, // no metadata
)
mem := memory.NewGoAllocator()
counter := int64(0)
b := array.NewRecordBuilder(mem, schema)
defer b.Release()
for i := 0; i < size; i++ {
b.Field(0).(*array.Int64Builder).AppendValues([]int64{counter}, nil)
counter++
b.Field(1).(*array.Int64Builder).AppendValues(
[]int64{int64(rand.Intn(100))}, nil)
b.Field(2).(*array.Int64Builder).AppendValues(
[]int64{int64(rand.Intn(100))}, nil)
b.Field(3).(*arrowArray.StringBuilder).AppendValues(
[]string{strconv.Itoa(rand.Intn(100))}, nil)
}
rec := b.NewRecord()
defer rec.Release()
tbl := array.NewTableFromRecords(schema, []arrowArray.Record{rec})
defer tbl.Release()
tr := array.NewTableReader(tbl, 1)
defer tr.Release()
data := []arrowArray.Record{}
for tr.Next() {
rec := tr.Record()
rec.Retain()
defer rec.Release()
data = append(data, rec)
}
if err := encodeDecode(data); err != nil {
fmt.Println("Finished with error", err)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment