a skeleton algorithm to batch sizable objects into predefined size limits in go language

makeBatch.go

package main

import (
	"fmt"
	"math/rand"
)

type arbitraryData struct {
	data []byte
}

func (dat arbitraryData) Size() int {
	return len(dat.data)
}

const MaxBatchSize = 32

func makeArbitraryData(length int) []arbitraryData {

	result := make([]arbitraryData, length)

	for idx := range result {
		result[idx].data = make([]byte, 1+rand.Int31n(MaxBatchSize))
		rand.Read(result[idx].data)
	}

	return result
}

func printBatch(batch []Batchable) string {
	res := ""
	sum := 0
	for _, dat := range batch {
		sum += dat.Size()
		res += fmt.Sprintf("%d, ", dat.Size())
	}
	return res + fmt.Sprintf(" = %d", sum)
}

type Batchable interface {
	Size() int
}

func BatchCollection(collection []Batchable, maxSize int, onReady func([]Batchable) error) error {

	lowerBound, upperBound := 0, 0

	dataIdx := 0

	for {

		bytesSoFar := 0

		if collection[dataIdx].Size() > maxSize {
			return fmt.Errorf("Chunk (%d) bigger than maximum batch chunk size (%d)", collection[dataIdx].Size(), maxSize)
		}

		for {
			if !(dataIdx < len(collection)) {
				break
			}

			if size := collection[dataIdx].Size(); bytesSoFar+size <= maxSize {
				bytesSoFar += size
				upperBound = dataIdx + 1
				dataIdx++
			} else {
				break
			}
		}

		err := onReady(collection[lowerBound:upperBound])

		if err != nil {
			return err
		}

		if upperBound >= len(collection) {
			break
		}

		lowerBound = upperBound
	}

	return nil
}

func main() {

	rand.Seed(259)

	arbData := makeArbitraryData(5)

	asBatchable := make([]Batchable, len(arbData))

	for idx := range arbData {
		asBatchable[idx] = interface{}(arbData[idx]).(Batchable)
	}

	fmt.Printf("Starting data: %+v\n", printBatch(asBatchable))

	batchIdx := 0

	BatchCollection(asBatchable, 32, func(batch []Batchable) error {
		fmt.Printf("Batch #%d: %+v\n", batchIdx, printBatch(batch))
		batchIdx++
		return nil
	})
}