Skip to content

Instantly share code, notes, and snippets.

@ZenGround0
Created January 3, 2018 20:26
Show Gist options
  • Save ZenGround0/49e4a1aa126736f966a1dfdcb84abdae to your computer and use it in GitHub Desktop.
Save ZenGround0/49e4a1aa126736f966a1dfdcb84abdae to your computer and use it in GitHub Desktop.
Golang HTTP multipart streaming
package main
import (
"io"
"mime/multipart"
"net/http"
"net/url"
"os"
"fmt"
)
const boundary = "MachliJalKiRaniHaiJeevanUskaPaaniHai"
func main() {
tr := http.DefaultTransport
client := &http.Client{
Transport: tr,
Timeout: 0,
}
fmt.Println("Set up pipe")
pR, pW := io.Pipe()
go func() {
// Set up multipart body for reading
multipartW := multipart.NewWriter(pW)
fmt.Println("Set up multipart writer")
multipartW.SetBoundary(boundary)
fmt.Println("Set up boundary")
partW, err0 := multipartW.CreateFormFile("fakefield", "fakefilename")
fmt.Println("Set up part writer")
if err0 != nil {
panic("Something is amiss creating a part")
}
connector := io.TeeReader(os.Stdin, partW)
buf := make([]byte, 256)
for {
/* stdin -> connector -> partW -> multipartW -> pW -> pR */
_, err := connector.Read(buf)
if err == io.EOF {
break
}
if err != nil {
fmt.Printf("The error reading from connector: %v", err)
}
}
}()
// Send http request chunk encoding the multipart message
req := &http.Request{
Method: "POST",
URL: &url.URL{
Scheme: "http",
Host: "localhost:9094",
Path: "/",
},
ProtoMajor: 1,
ProtoMinor: 1,
ContentLength: -1,
Body: pR,
}
fmt.Printf("Doing request\n")
_, err := client.Do(req)
fmt.Printf("Done request. Err: %v\n", err)
}
package main
import (
"mime/multipart"
"net/http"
"net"
"fmt"
"io"
)
const boundary = "MachliJalKiRaniHaiJeevanUskaPaaniHai"
func handle(w http.ResponseWriter, req *http.Request) {
partReader := multipart.NewReader(req.Body, boundary)
buf := make([]byte, 256)
for {
part, err := partReader.NextPart()
if err == io.EOF {
break
}
var n int
for {
n, err = part.Read(buf)
if err == io.EOF {
break
}
fmt.Printf(string(buf[:n]))
}
fmt.Printf(string(buf[:n]))
}
}
func main() {
/* Net listener */
n := "tcp"
addr := "127.0.0.1:9094"
l, err := net.Listen(n, addr)
if err != nil {
panic("AAAAH")
}
/* HTTP server */
server := http.Server{
Handler: http.HandlerFunc(handle),
}
server.Serve(l)
}
@ZenGround0
Copy link
Author

A basic proof of concept that golang http streaming works with the multipart content type. To see this compile and build both files in their own packages. Make sure no process is using port 9094. Start ./server in one terminal and ./client in another. The client sends a multipart message with a single part, and the body of this part is read from standard in. Enter data into the client's standard in and watch as the server processes the available data of the part without first reading the entire part. Best of all this works with the mime/multipart standard library for parsing multipart requests.

@robtimus
Copy link

robtimus commented Dec 7, 2018

Instead of the panics while copying data in the goroutine, you should use pW.CloseWithError. This will make the reader return an error when attempting to read from it.

You should probably also close pW when there's nothing to read anymore.

@patientplatypus
Copy link

patientplatypus commented Jan 27, 2019

This code doesn't run (correctly). The go function in the client fires after the request is already sent (because go functions are async) and the request ends up sending nothing. I've tested this. Please modify your code.

@codekoala
Copy link

Thank you for this! It worked well for me after I added the suggestions from @robtimus

diff --git a/client.go b/client.go
index a9fb70b..642cba8 100644
--- a/client.go
+++ b/client.go
@@ -1,12 +1,12 @@
 package main

 import (
+       "fmt"
        "io"
        "mime/multipart"
        "net/http"
        "net/url"
        "os"
-       "fmt"
 )

 const boundary = "MachliJalKiRaniHaiJeevanUskaPaaniHai"
@@ -31,7 +31,7 @@ func main() {
                partW, err0 := multipartW.CreateFormFile("fakefield", "fakefilename")
                fmt.Println("Set up part writer")
                if err0 != nil {
-                       panic("Something is amiss creating a part")
+                       pW.CloseWithError(err0)
                }

                connector := io.TeeReader(os.Stdin, partW)
@@ -40,13 +40,14 @@ func main() {
                        /* stdin -> connector -> partW -> multipartW -> pW -> pR */
                        _, err := connector.Read(buf)
                        if err == io.EOF {
+                               multipartW.Close()
+                               pW.Close()
                                break
                        }
                        if err != nil {
                                fmt.Printf("The error reading from connector: %v", err)
                        }
                }
-
        }()

        // Send http request chunk encoding the multipart message

@aderbedr
Copy link

aderbedr commented May 8, 2024

@patientplatypus You need to pipe data into it, akin to

cat file.txt | ./client

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment