io.Reader & io.Writer fun

01-curl.go

package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
)

func init() {
	if len(os.Args) != 2 {
		fmt.Println("Usage: go run 01-curl.go <url>")
		os.Exit(-1)
	}
}

// We start with a simplistic version of curl.  Run it with a URL, and it
// downloads the URL and writes it to stdout.
func main() {
	// r here is a response, and r.Body is an io.Reader
	r, err := http.Get(os.Args[1])
	if err != nil {
		fmt.Println(err)
		return
	}
	// io.Copy(dst io.Writer, src io.Reader), copies from the Body to Stdout
	io.Copy(os.Stdout, r.Body)
	if err = r.Body.Close(); err != nil {
		fmt.Println(err)
	}
}

02-curl.go

package main

import (
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
)

// Let's implement 2 more features of curl;  One is to send the output to a file,
// And the other is to squelch printing to Stdout.  Unlike curl, we'll make these
// independent;  we'll always output to Stdout unless silent is true.
var Config struct {
	Silent   bool
	DestFile string
}

func init() {
	// Let the flag package handle the options;  -o for output and -s for silent
	flag.StringVar(&Config.DestFile, "o", "", "output file")
	flag.BoolVar(&Config.Silent, "s", false, "silent (do not output to stdout)")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Println("Usage: go run 02-curl.go [options] <url>")
		os.Exit(-1)
	}
}

func main() {
	r, err := http.Get(flag.Args()[0])
	if err != nil {
		fmt.Println(err)
		return
	}

	// this is a slice of io.Writers we will write the file to
	var writers []io.Writer

	// if we aren't in Silent mode, lets add Stdout to our writers
	if !Config.Silent {
		writers = append(writers, os.Stdout)
	}

	// if DestFile was provided, lets try to create it and add to the writers
	if len(Config.DestFile) > 0 {
		file, err := os.Create(Config.DestFile)
		if err != nil {
			fmt.Println(err)
			return
		}
		writers = append(writers, file)
		defer file.Close()
	}

	// MultiWriter(io.Writer...) returns a single writer which multiplexes its
	// writes across all of the writers we pass in.
	dest := io.MultiWriter(writers...)

	// write to dest the same way as before, copying from the Body
	io.Copy(dest, r.Body)
	if err = r.Body.Close(); err != nil {
		fmt.Println(err)
	}
}

03-curl.go

package main

import (
	"compress/gzip"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
)

// That was easy!  Let's add another few features.  If -z is passed, we want any
// DestFile's to be gzipped.  If -md5 is passed, we want print the md5sum of the
// data that's been transfered instead of the data itself.
var Config struct {
	Silent   bool
	DestFile string
	Gzip     bool
	Md5      bool
}

func init() {
	flag.StringVar(&Config.DestFile, "o", "", "output file")
	flag.BoolVar(&Config.Silent, "s", false, "silent (do not output to stdout)")
	flag.BoolVar(&Config.Gzip, "z", false, "gzip file output")
	flag.BoolVar(&Config.Md5, "md5", false, "stdout md5sum instead of body")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Println("Usage: go run 03-curl.go [options] <url>")
		os.Exit(-1)
	}
}

func main() {
	url := flag.Args()[0]
	r, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return
	}

	// Our Md5 hash destination, which is an io.Writer that computes the
	// hash of whatever is written to it.
	hash := md5.New()
	var writers []io.Writer

	// if we aren't in Silent mode, we've got to output something
	if !Config.Silent {
		// If -md5 was passed, write to the hash instead of os.Stdout
		if Config.Md5 {
			writers = append(writers, hash)
		} else {
			writers = append(writers, os.Stdout)
		}
	}

	// if DestFile was provided, we've got to write a file
	if len(Config.DestFile) > 0 {
		// by declaring writer here as a WriteCloser, we're saying that we don't care
		// what the underlying implementation will be, all we require is something that
		// can Write and Close;  both os.File and the gzip.Writer are WriteClosers.
		var writer io.WriteCloser
		writer, err := os.Create(Config.DestFile)
		if err != nil {
			fmt.Println(err)
			return
		}
		// If we're in Gzip mode, wrap the writer in gzip
		if Config.Gzip {
			writer = gzip.NewWriter(writer)
		}
		writers = append(writers, writer)
		defer writer.Close()
	}

	// MultiWriter(io.Writer...) returns a single writer which multiplexes its
	// writes across all of the writers we pass in.
	dest := io.MultiWriter(writers...)

	// write to dest the same way as before, copying from the Body
	io.Copy(dest, r.Body)
	if err = r.Body.Close(); err != nil {
		fmt.Println(err)
		return
	}

	// finally, if we were in Md5 output mode, lets output the checksum and url:
	if Config.Md5 {
		fmt.Printf("%x  %s\n", hash.Sum(nil), url)
	}
}

README.md

This repository has a few examples within that are designed to show off not only the simplicity, flexibility, and power of io.Reader and io.Writer, but also the way that Go enables us to build powerful programs from simple components via composition.

Unusually for a Go repository, it's not meant to be installed with go get; each file in this repos is a stand-alone program using only the standard library, designed to be run in isolation with go run and inspected and/or modified as a learning exercise. Writing it was, itself, a learning exercise.

The files should be read first, understood, and then run.

Oh, I've been working with go since years, but never saw that clever trick with io.MultiWriter for multiplexing before. Neat! Thanks:)

Nice job

thanks!

Thanks!

thanks

Hi there! The Humanitarian Organisation for Migration Economics (HOME) has created an essential platform, My Voice Home https://myvoiceathome.org/ to amplify the voices of migrant workers in Singapore. Since 2004, HOME has worked to protect and advocate for these workers, especially those affected by trafficking or exploitation. Through this platform, migrant workers share their stories, poetry, and experiences, fostering connection and raising awareness.