Line by line compression vs compressing all the content using gzip

main.go

package main

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"time"
)

func main() {
	// Download from https://corpus.canterbury.ac.nz/descriptions/
	inFile, err := os.Open("world192.txt")
	if err != nil {
		panic(err)
	}

	if err := streamAll(inFile); err != nil {
		panic(err)
	}
	_, _ = inFile.Seek(0, 0)
	if err := streamLine(inFile); err != nil {
		panic(err)
	}
}

func streamAll(file *os.File) error {
	opFile, err := os.Create("file.txt.streamALl.gzip")
	if err != nil {
		return err
	}
	start := time.Now()
	w := gzip.NewWriter(opFile)
	bytes, err := io.ReadAll(file)
	if err != nil {
		return err
	}
	_, err = w.Write(bytes)
	if err != nil {
		return err
	}
	w.Flush()
	since := time.Since(start)
	inStat, err := os.Stat(file.Name())
	if err != nil {
		return err
	}
	opStat, err := os.Stat(opFile.Name())
	if err != nil {
		return err
	}

	fmt.Println("StreamAll", "time: ", since, "compression ratio:", float64(inStat.Size())/float64(opStat.Size()))

	return nil
}

func streamLine(file *os.File) error {
	opFile, err := os.Create("file.txt.streamLines.gzip")
	if err != nil {
		return err
	}
	start := time.Now()
	w := gzip.NewWriter(opFile)
	reader := bufio.NewReader(file)
	for {
		bytes, err := reader.ReadBytes('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		if _, err := w.Write(bytes); err != nil {
			return err
		}
		w.Flush()

	}
	since := time.Since(start)
	inStat, err := os.Stat(file.Name())
	if err != nil {
		return err
	}
	opStat, err := os.Stat(opFile.Name())
	if err != nil {
		return err
	}

	fmt.Println("StreamLine", "time: ", since, "compression ratio:", float64(inStat.Size())/float64(opStat.Size()))

	return nil
}