-
-
Save amustafa/27ffc4c8debaeb05dbdf9608854bf670 to your computer and use it in GitHub Desktop.
This is an example script that causes Apache Arrow ipc.Writer in the Go library to panic. (https://pkg.go.dev/github.com/apache/arrow/go/arrow@v0.0.0-20210127213842-4a9885e30a98/ipc#Writer)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
panic: arrow/array: index out of range | |
goroutine 1 [running]: | |
github.com/apache/arrow/go/arrow/array.NewSliceData(0xc000182a20, 0x8, 0x1, 0x8904008) | |
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/array/data.go:149 +0x1c6 | |
github.com/apache/arrow/go/arrow/ipc.(*recordEncoder).visit(0xc0001afa30, 0xc0001afa00, 0x1169180, 0xc000186740, 0x0, 0x0) | |
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/ipc/writer.go:273 +0x15c5 | |
github.com/apache/arrow/go/arrow/ipc.(*recordEncoder).Encode(0xc0001afa30, 0xc0001afa00, 0x11696c0, 0xc00018e2d0, 0x10, 0x1773fd8) | |
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/ipc/writer.go:178 +0x285 | |
github.com/apache/arrow/go/arrow/ipc.(*Writer).Write(0xc000186b00, 0x11696c0, 0xc00018e2d0, 0x0, 0x0) | |
/Users//opt/go/pkg/mod/github.com/apache/arrow/go/arrow@v0.0.0-20210318134220-775a714006e2/ipc/writer.go:130 +0x1a8 | |
main.encodeDataArrow(0x11696c0, 0xc00018e2d0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
/Users//workspace/record_serialize.go:46 +0x14c | |
main.encodeDecode(0xc0001869c0, 0x3, 0x4) | |
/Users//workspace/record_serialize.go:69 +0x219 | |
main.main() | |
/Users//workspace/record_serialize.go:127 +0x793 | |
exit status 2 | |
*/ | |
package main | |
import ( | |
"bytes" | |
"errors" | |
"fmt" | |
"math/rand" | |
"strconv" | |
"github.com/apache/arrow/go/arrow" | |
"github.com/apache/arrow/go/arrow/array" | |
arrowArray "github.com/apache/arrow/go/arrow/array" | |
arrowIpc "github.com/apache/arrow/go/arrow/ipc" | |
"github.com/apache/arrow/go/arrow/memory" | |
) | |
func encodeDataArrow(rec arrowArray.Record) ([]byte, error) { | |
rec.Retain() | |
defer rec.Release() | |
var ipcOutput bytes.Buffer | |
w := arrowIpc.NewWriter(&ipcOutput, arrowIpc.WithSchema(rec.Schema())) | |
defer w.Close() | |
if err := w.Write(rec); err != nil { | |
return nil, err | |
} | |
b := ipcOutput.Bytes() | |
return b, nil | |
} | |
func decodeArrow(data []byte) (arrowArray.Record, error) { | |
buf := bytes.NewBuffer(data) | |
reader, err := arrowIpc.NewReader(buf) | |
if err != nil { | |
return nil, err | |
} | |
for reader.Next() { | |
rec := reader.Record() | |
rec.Retain() | |
return rec, nil | |
} | |
return nil, errors.New("didn't get any records") | |
} | |
func encodeDecode(data []arrowArray.Record) error { | |
for _, record := range data { | |
bytes, err := encodeDataArrow(record) | |
if err != nil { | |
return err | |
} | |
deserializedData, err := decodeArrow(bytes) | |
if err != nil { | |
return err | |
} | |
fmt.Println(deserializedData) | |
} | |
return nil | |
} | |
func main() { | |
size := 3 | |
schema := arrow.NewSchema( | |
[]arrow.Field{ | |
arrow.Field{Name: "idx", Type: arrow.PrimitiveTypes.Int64}, | |
arrow.Field{Name: "A", Type: arrow.PrimitiveTypes.Int64}, | |
arrow.Field{Name: "B", Type: arrow.PrimitiveTypes.Int64}, | |
arrow.Field{Name: "C", Type: arrow.BinaryTypes.String}, | |
}, | |
nil, // no metadata | |
) | |
mem := memory.NewGoAllocator() | |
counter := int64(0) | |
b := array.NewRecordBuilder(mem, schema) | |
defer b.Release() | |
for i := 0; i < size; i++ { | |
b.Field(0).(*array.Int64Builder).AppendValues([]int64{counter}, nil) | |
counter++ | |
b.Field(1).(*array.Int64Builder).AppendValues( | |
[]int64{int64(rand.Intn(100))}, nil) | |
b.Field(2).(*array.Int64Builder).AppendValues( | |
[]int64{int64(rand.Intn(100))}, nil) | |
b.Field(3).(*arrowArray.StringBuilder).AppendValues( | |
[]string{strconv.Itoa(rand.Intn(100))}, nil) | |
} | |
rec := b.NewRecord() | |
defer rec.Release() | |
tbl := array.NewTableFromRecords(schema, []arrowArray.Record{rec}) | |
defer tbl.Release() | |
tr := array.NewTableReader(tbl, 1) | |
defer tr.Release() | |
data := []arrowArray.Record{} | |
for tr.Next() { | |
rec := tr.Record() | |
rec.Retain() | |
defer rec.Release() | |
data = append(data, rec) | |
} | |
if err := encodeDecode(data); err != nil { | |
fmt.Println("Finished with error", err) | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment