Skip to content

Instantly share code, notes, and snippets.

@rodaine
Last active March 14, 2021 21:25
Show Gist options
  • Save rodaine/a4236a51b1a3b66d1e08 to your computer and use it in GitHub Desktop.
Save rodaine/a4236a51b1a3b66d1e08 to your computer and use it in GitHub Desktop.
Code snippets for my blog post "Asynchronously Split an io.Reader in Go" (http://rodaine.com/2015/04/async-split-io-reader-in-golang/)
func handleUpload(u io.Reader) (err error) {
// capture all bytes from upload
b, err := ioutil.ReadAll(u)
if err != nil {
return
}
// wrap the bytes in a ReadSeeker
r := bytes.NewReader(b)
// process the meta data
err = processMetaData(r)
if err != nil {
return
}
// rewind the reader back to the start
r.Seek(0, 0)
// upload the data
err = uploadFile(r)
if err != nil {
return
}
return nil
}
func handleUpload(u io.Reader) (err error) {
// create a temporary file for the upload
f, err := ioutil.TempFile("", "upload")
if err != nil {
return
}
// destroy the file once done
defer func() {
n := f.Name()
f.Close()
os.Remove(n)
}()
// transfer the bytes to the file
_, err = io.Copy(f, u)
if err != nil {
return
}
// rewind the file
f.Seek(0, 0)
// process the meta data
err = processMetaData(f)
if err != nil {
return
}
// rewind the file again
f.Seek(0, 0)
// upload the file
err = uploadFile(f)
if err != nil {
return
}
return nil
}
func handleUpload(u io.Reader) (err error) {
// read in the first two bytes
b := make([]byte, 2)
_, err = u.Read(b)
if err != nil {
return
}
// check that they match the JPEG header
jpg := []byte{0xFF, 0xD8}
if !bytes.Equal(b, jpg) {
return errors.New("not a JPEG")
}
// glue those bytes back onto the reader
r := io.MultiReader(bytes.NewReader(b), u)
// upload the file
err = uploadFile(r)
if err != nil {
return
}
return nil
}
func handleUpload(u io.Reader) {
// create the pipes
mp4R, mp4W := io.Pipe()
webmR, webmW := io.Pipe()
oggR, oggW := io.Pipe()
wavR, wavW := io.Pipe()
// create channel to synchronize
done := make(chan bool)
defer close(done)
// spawn all the task goroutines. These look identical to
// the TeeReader example, but pulled out into separate
// methods for clarity
go uploadMP4(mp4R, done)
go transcodeAndUploadWebM(webmR, done)
go transcodeAndUploadOgg(oggR, done)
go transcodeAndUploadWav(wavR, done)
go func() {
// after completing the copy, we need to close
// the PipeWriters to propagate the EOF to all
// PipeReaders to avoid deadlock
defer mp4W.Close()
defer webmW.Close()
defer oggW.Close()
defer wavW.Close()
// build the multiwriter for all the pipes
mw := io.MultiWriter(mp4W, webmW, oggW, wavW)
// copy the data into the multiwriter
io.Copy(mw, u)
}()
// wait until all are done
for c := 0; c < 4; c++ {
<-done
}
}
func handleUpload(u io.Reader) {
// create the pipe and tee reader
pr, pw := io.Pipe()
tr := io.TeeReader(u, pw)
// create channel to synchronize
done := make(chan bool)
defer close(done)
go func() {
// close the PipeWriter after the
// TeeReader completes to trigger EOF
defer pw.Close()
// upload the original MP4 data
uploadFile(tr)
done <- true
}()
go func() {
// transcode to WebM
webmr := transcode(pr)
// upload to storage
uploadFile(webmr)
done <- true
}()
// wait until both are done
for c := 0; c < 2; c++ {
<-done
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment