unmarshal_large_dataset_optimal.go

func readBookDatasetOptimized(protoInFile string) error {
	var rLen uint32
	var book typedefs.Book
  //Buffer to read bytes for decoding
	var buf bytes.Buffer
	done := false
  //Fixed Length Buffer for reading from the file
	rbuf := make([]byte, 1024)
	r, err := os.Open(protoInFile)
	if err != nil {
		log.Fatalf("Failed to read file %s", err.Error())
	}
	defer r.Close()
	for {
		if !done {
			err := fillBuffer(r, &buf, rbuf)
			done = err == io.EOF

		}
		if done && buf.Len() == 0 {
			return nil
		}
		if done && buf.Len() < 4 {
			return fmt.Errorf("Bad encoding, unexpected format")
		}

		rLen = binary.LittleEndian.Uint32(buf.Next(4))
		if done && buf.Len() < int(rLen) {
			return fmt.Errorf("Bad encoding, buf len %d where as eof reached", buf.Len())
		}
		for !done && buf.Len() < int(rLen) {
			err := fillBuffer(r, &buf, rbuf)
			done = err == io.EOF
		}

		if err := proto.Unmarshal(buf.Next(int(rLen)), &book); err != nil {
			return err
		}
		fmt.Printf("Title: %s \nAuthor: %s \nISBN: %s \nOverview: %s\n\n", book.Title, book.Author, book.Isbn, book.Overview)

	}

}
//Transfer bytes read from the given reader(source) to the bytes.Buffer(target)
func fillBuffer(src io.Reader, target *bytes.Buffer, rbuf []byte) error {
	n, err := src.Read(rbuf)
	target.Write(rbuf[:n])
	return err
}