Skip to content

Instantly share code, notes, and snippets.

@ahamilton55
Created August 18, 2017 19:27
Show Gist options
  • Save ahamilton55/6a3faf3a996d029d4e269d9f0d9e2e85 to your computer and use it in GitHub Desktop.
Save ahamilton55/6a3faf3a996d029d4e269d9f0d9e2e85 to your computer and use it in GitHub Desktop.
package main
import (
"bytes"
"fmt"
"io/ioutil"
"log"
"github.com/ahamilton55/zapier-test/mcchunker"
"github.com/bradfitz/gomemcache/memcache"
)
func main() {
dat, err := ioutil.ReadFile("bigoldfile.dat")
if err != nil {
log.Fatalf("unable to read data: %v\n", err)
}
mc := memcache.New("127.0.0.1:11211")
key := "test_key"
if err := mcchunker.Send(mc, key, dat); err != nil {
log.Fatalf("unable to set data: %v\n", err)
}
chkData, err := mcchunker.Get(mc, key)
if err != nil {
log.Fatalf("unable to get data: %v\n", err)
}
fmt.Printf("Original length: %d\nStored length: %d\n", len(dat), len(chkData))
if bytes.Compare(dat, chkData) != 0 {
log.Fatalf("incorrect data returned")
}
fmt.Println("Success")
mc.FlushAll()
}
package mcchunker
import (
"fmt"
"math"
"github.com/bradfitz/gomemcache/memcache"
)
const (
megabyte = 1.0 * 1000 * 1000
keyFmt = "%s_%d"
)
// MCChunker is a helper type for storing the memcached client
type MCChunker struct {
mc *memcache.Client
}
// New will produce a
func New(mc *memcache.Client) *MCChunker {
return &MCChunker{mc: mc}
}
// Send a file to memcached and chunk up the file if neccessary
func Send(mc *memcache.Client, key string, data []byte) error {
var err error
lower := 0
if len(data) > megabyte {
maxChunk := int(math.Ceil(float64(len(data)) / megabyte))
for i := 0; i < maxChunk; i++ {
upper := ((i + 1) * int(megabyte)) - 1
if upper >= len(data) {
upper = len(data)
}
err = mc.Set(&memcache.Item{
Key: fmt.Sprintf(keyFmt, key, i),
Value: data[lower:upper],
})
if err != nil {
return err
}
lower = upper
}
} else {
err = mc.Set(&memcache.Item{Key: key, Value: data})
if err != nil {
return err
}
}
return nil
}
// Get a file from memcached, if needed, get the chunks of a file
func Get(mc *memcache.Client, key string) ([]byte, error) {
item, err := mc.Get(key)
if err == nil {
return item.Value, nil
}
var data []byte
for i := 0; true; i++ {
item, err := mc.Get(fmt.Sprintf(keyFmt, key, i))
if err != nil && i == 0 {
return nil, err
} else if err != nil {
break
}
data = append(data, item.Value...)
}
return data, nil
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment