Exploration of processing writes with batching, throttling, and a global timeout

main.go

package main

import (
	"fmt"
	"time"
)

/* The requirements are:
1. process the list in batches
2. take into account unprocessed items in the dynamo response
3. always breathe between requests to dynamo
4. give up if it takes too long overall */

func writeToDynamo(items []int, batchSize int) ([]int, error) {
	var batch []int

	if len(items) < batchSize {
		batch = items
	} else {
		batch = items[0:batchSize]
	}

	fmt.Printf("writing %v\n", batch) // pretend we write them

	return items[batchSize:], nil
}

type Options struct {
	batchSize int
	thottle   time.Duration
	timeout   <-chan time.Time
}

func process(items []int, options Options) error {
	remaining := items
	var err error

	for len(remaining) > 0 && err == nil {
		select {
		case <-options.timeout:
			return fmt.Errorf("processing stopped as hit timeout")
		case <-time.After(options.thottle):
			remaining, err = writeToDynamo(remaining, options.batchSize)
		}
	}

	return err
}

func main() {
	items := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}

	options := Options{
		batchSize: 2,
		thottle:   time.Second * 2,
		timeout:   time.After(time.Second * 8),
	}

	err := process(items, options)
	if err != nil {
		fmt.Println(err.Error())
	}
}