serialize_large_payload.go

func serializeBookDataset(protoOutFile string) error {
	w, err := os.OpenFile(protoOutFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer w.Close()

	csvFile, err := os.Open("books.csv")
	if err != nil {
		return err
	}
	defer csvFile.Close()

	r := csv.NewReader(csvFile)
	count := 0
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}		
		out, err := proto.Marshal(&typedefs.Book{Title: record[1], Author: record[2], Isbn: record[9], Overview: record[19]})
		if err != nil {
			return err
		}
		//Encode length as []byte and append the message []byte
		l := uint32(len(out))
		buf := make([]byte, 4)
		binary.LittleEndian.PutUint32(buf[0:], l)
		buf = append(buf, out...)

		_, err = w.Write(buf)
		if err != nil {
			return err
		}
		count++

	}
	fmt.Println("Total Records written ", count)

	return nil
}