Simulate a large payload on the fly with an io.Reader

main.go

package main

import (
	"fmt"
	"io"
)

func main() {
	r := NewDataGenerator(1000)

	var err error
	var total int

	buffer := make([]byte, 120)
	for {
		var n int
		n, err = r.Read(buffer)
		total = total + n
		fmt.Println(fmt.Sprintf("Reading %d bytes. Total: %d", n, total))

		if err == io.EOF {
			break
		}
	}
}

type DataGenerator struct {
	total   int64
	current int64
}

func NewDataGenerator(total int64) *DataGenerator {
	return &DataGenerator{
		total:   total,
		current: 0,
	}
}

func (dg *DataGenerator) eof() bool {
	return dg.current >= dg.total
}

func (dg *DataGenerator) readByte() byte {
	// this function assumes that eof() check was done before
	dg.current = dg.current + int64(1)
	return byte(1)
}

func (dg *DataGenerator) Read(p []byte) (n int, err error) {
	if dg.eof() {
		err = io.EOF
		return
	}

	if c := cap(p); c > 0 {
		for n < c {
			p[n] = dg.readByte()
			n++
			if dg.eof() {
				break
			}
		}
	}
	return
}