Skip to content

Instantly share code, notes, and snippets.

@nolash
Created November 28, 2017 19:31
Show Gist options
  • Save nolash/b215bb2c979e829ff5fa8010302a3c6c to your computer and use it in GitHub Desktop.
Save nolash/b215bb2c979e829ff5fa8010302a3c6c to your computer and use it in GitHub Desktop.
swarm dpa bigfile test
package main
import (
"crypto/rand"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"sync"
sys "golang.org/x/sys/unix"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage"
)
const (
EXTRA_BYTES = 1024 * 1024
)
var (
datasize int64
datadir string
pagesize int
keep bool
)
func init() {
flag.Int64Var(&datasize, "s", 0, "size of data to write")
flag.StringVar(&datadir, "d", ".", "dir to write to")
flag.IntVar(&pagesize, "p", 4096, "size of pages to read back from dpa")
flag.BoolVar(&keep, "k", false, "do not delete datadir")
var verbose bool
flag.BoolVar(&verbose, "v", false, "output debug information")
var veryverbose bool
flag.BoolVar(&veryverbose, "vv", false, "output a lot more debug information")
flag.Parse()
loglevel := log.LvlWarn
if veryverbose {
loglevel = log.LvlTrace
} else if verbose {
loglevel = log.LvlDebug
}
//log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(loglevel, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(loglevel, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))))
}
func main() {
if datasize == 0 {
fmt.Fprintf(os.Stderr, "ERROR: -s required!\n")
flag.Usage()
os.Exit(1)
}
var stat sys.Statfs_t
sys.Statfs(datadir, &stat)
bytesavail := (uint64(stat.Bsize) * stat.Bavail) + EXTRA_BYTES
if bytesavail < uint64(datasize+EXTRA_BYTES) {
fmt.Printf("Need %dB, have %dB\n", datasize, bytesavail)
os.Exit(1)
}
dir, err := ioutil.TempDir(datadir, fmt.Sprintf("dpabig-%d-", datasize))
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if !keep {
defer os.RemoveAll(dir)
}
// in dbcapacity; allow for indices too, 1+64+8+8 bytes per chunk, adds approx 1/40 of volume
dpa, err := storage.NewLocalDPA(dir, "SHA3", uint64((datasize/storage.CHUNKSIZE)*1.1), 5000)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
dpa.Start()
defer dpa.Stop()
swg := &sync.WaitGroup{}
wwg := &sync.WaitGroup{}
key, err := dpa.Store(rand.Reader, datasize, swg, wwg)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
log.Debug("stored", "key", key)
r := dpa.Retrieve(key)
dpasize, err := r.Size(nil)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
log.Debug("dpa size", "bytes", dpasize)
dataout := make([]byte, pagesize)
bytesread := 0
for {
c, err := r.Read(dataout)
bytesread += c
if err != nil {
if err != io.EOF {
log.Error("read fail", "err", err)
}
break
}
}
fmt.Printf("bytes read: %d\n", bytesread)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment