Skip to content

Instantly share code, notes, and snippets.

@ericjster
Created May 16, 2019 07:21
Show Gist options
  • Save ericjster/eea32820660e26b16174fadd4034dd9d to your computer and use it in GitHub Desktop.
Save ericjster/eea32820660e26b16174fadd4034dd9d to your computer and use it in GitHub Desktop.
Golang write binary file using memory mapped structs
package main
// Example of writing a binary file of float structs.
import (
"log"
"os"
"reflect"
"testing"
"unsafe"
mmapLib "github.com/edsrzf/mmap-go"
)
// Struct definition for disk layout.
type myOutputPoint struct {
X float32
Y float32
}
const tmpMyBinaryDataFileName = "/tmp/binaryDataFile.bin"
// Write a data file having an array of structs. Use memory mapped IO.
func writeDataMMapStructArr(fname string, x []float64, y []float64, limitSample int) {
n := len(x)
if limitSample > 0 && n > limitSample {
n = limitSample
}
nBytes := n * int(unsafe.Sizeof(outputPoint{}))
// Create new file
f, err := os.Create(fname)
if err != nil {
log.Fatalf("error creating: %s, %v", fname, err)
}
defer f.Close()
// Resize
err = f.Truncate(int64(nBytes))
if err != nil {
log.Fatalf("error resizing: %s, %v", fname, err)
}
mmap, err := mmapLib.Map(f, os.O_RDWR, 0)
if err != nil {
log.Fatalf("error mapping: %v", err)
}
defer mmap.Unmap()
// Slice of []myOutputPoint pointing into mmap.
var dataDst []myOutputPoint
shDst := (*reflect.SliceHeader)(unsafe.Pointer(&dataDst))
shDst.Data = uintptr(unsafe.Pointer(&mmap[0]))
shDst.Len = n
shDst.Cap = n
for i := range dataDst {
dataDst[i].X = float32(x[i])
dataDst[i].Y = float32(y[i])
}
}
const myNumPointsForBenchmark = 1e6
func myPopulateTestData(n int) ([]float64, []float64) {
x := make([]float64, n)
y := make([]float64, n)
for i := 0; i < n; i++ {
x[i] = float64(i) / 1.e6
y[i] = float64(i)
}
return x, y
}
func BenchmarkWriteDataMMapStructArr(b *testing.B) {
x, y := myPopulateTestData(myNumPointsForBenchmark)
for n := 0; n < b.N; n++ {
writeDataMMapStructArr(tmpMyBinaryDataFileName, x, y, 0)
}
}
@dzpt
Copy link

dzpt commented Jul 5, 2021

this is slide of struct, not to map a single struct?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment