The python script used minio1
to perform the multipart upload.
minio2
was brought down and back up while many concurrent multipart uploads were ongoing.
➜ ~ tree /mnt/export/minio{1..4}/.minio.sys/multipart/kp-test-minio-py/my-boring-object-3/
/mnt/export/minio1/.minio.sys/multipart/kp-test-minio-py/my-boring-object-3/
└── cf06534c-bcee-405c-976d-b22a3bdb71a7
└── part.10
/mnt/export/minio2/.minio.sys/multipart/kp-test-minio-py/my-boring-object-3/
├── cf06534c-bcee-405c-976d-b22a3bdb71a7
│ ├── part.1
│ ├── part.2
│ ├── part.3
│ └── xl.json
└── uploads.json
/mnt/export/minio3/.minio.sys/multipart/kp-test-minio-py/my-boring-object-3/
└── cf06534c-bcee-405c-976d-b22a3bdb71a7
└── part.10
/mnt/export/minio4/.minio.sys/multipart/kp-test-minio-py/my-boring-object-3/
└── cf06534c-bcee-405c-976d-b22a3bdb71a7
└── part.10
- []xlMetaV1
[
{ {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []}
{ {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []}
{ {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []}
{ {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []}
]
- modTime
0001-01-01 00:00:00 +0000 UTC
minio2_1 | Status: 4 Online, 0 Offline. We can withstand [2] more drive failure(s).
minio1_1 | 2016/11/17 13:33:14 http: panic serving 172.19.0.1:60810: Unable to look for valid XL metadata content - [{ {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []} { {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []} { {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []} { {0 0001-01-01 00:00:00 +0000 UTC} { 0 0 0 0 [] []} {} map[] []}] 0001-01-01 00:00:00 +0000 UTC
minio1_1 | goroutine 52906 [running]:
minio1_1 | net/http.(*conn).serve.func1(0xc420376d00)
minio1_1 | /usr/local/go/src/net/http/server.go:1491 +0x12a
minio1_1 | panic(0xa76be0, 0xc428eaa4c0)
minio1_1 | /usr/local/go/src/runtime/panic.go:458 +0x243
minio1_1 | github.com/minio/minio/cmd.pickValidXLMeta(0xc420206a80, 0x4, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
minio1_1 | /go/src/github.com/minio/minio/cmd/xl-v1-metadata.go:206 +0x3ae
minio1_1 | github.com/minio/minio/cmd.xlObjects.PutObjectPart(0xc42022bde8, 0xc420229e40, 0x4, 0x4, 0x2, 0x2, 0x2, 0x3, 0xc4204bade0, 0xc420229e80, ...)
minio1_1 | /go/src/github.com/minio/minio/cmd/xl-v1-multipart.go:496 +0x1071
minio1_1 | github.com/minio/minio/cmd.(*xlObjects).PutObjectPart(0xc420227320, 0xc422cdaf55, 0x10, 0xc422cdaf66, 0x12, 0xc422cdaf90, 0x24, 0xa, 0x500000, 0x7fc7a7784140, ...)
minio1_1 | <autogenerated>:854 +0x15a
minio1_1 | github.com/minio/minio/cmd.objectAPIHandlers.PutObjectPartHandler(0xbdcbf8, 0xee3180, 0xc427a61450, 0xc4202f6c30)
minio1_1 | /go/src/github.com/minio/minio/cmd/object-handlers.go:688 +0x6b1
minio1_1 | github.com/minio/minio/cmd.(objectAPIHandlers).PutObjectPartHandler-fm(0xee3180, 0xc427a61450, 0xc4202f6c30)
minio1_1 | /go/src/github.com/minio/minio/cmd/api-router.go:44 +0x4e
minio1_1 | net/http.HandlerFunc.ServeHTTP(0xc4203a6790, 0xee3180, 0xc427a61450, 0xc4202f6c30)
minio1_1 | /usr/local/go/src/net/http/server.go:1726 +0x44
minio1_1 | github.com/minio/minio/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc4202f8230, 0xee3180, 0xc427a61450, 0xc4202f6c30)
minio1_1 | /go/src/github.com/minio/minio/vendor/github.com/gorilla/mux/mux.go:107 +0x10d
minio1_1 | github.com/minio/minio/cmd.requestSizeLimitHandler.ServeHTTP(0xedc580, 0xc4202f8230, 0x180000000, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/generic-handlers.go:60 +0x100
minio1_1 | github.com/minio/minio/cmd.(*requestSizeLimitHandler).ServeHTTP(0xc4203cb9c0, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:199 +0x82
minio1_1 | github.com/minio/minio/cmd.crossDomainPolicy.ServeHTTP(0xedea40, 0xc4203cb9c0, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/crossdomain-xml-handler.go:48 +0x65
minio1_1 | github.com/minio/minio/cmd.(*crossDomainPolicy).ServeHTTP(0xc4203daf50, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:135 +0x79
minio1_1 | github.com/minio/minio/cmd.redirectHandler.ServeHTTP(0xede900, 0xc4203daf50, 0xb72dc2, 0x6, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/generic-handlers.go:97 +0xb9
minio1_1 | github.com/minio/minio/cmd.(*redirectHandler).ServeHTTP(0xc4203cb9e0, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:200 +0x8b
minio1_1 | github.com/minio/minio/cmd.minioPrivateBucketHandler.ServeHTTP(0xedea00, 0xc4203cb9e0, 0xb72dc2, 0x6, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/generic-handlers.go:145 +0xb0
minio1_1 | github.com/minio/minio/cmd.(*minioPrivateBucketHandler).ServeHTTP(0xc4203cba00, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:202 +0x8b
minio1_1 | github.com/minio/minio/cmd.cacheControlHandler.ServeHTTP(0xede9c0, 0xc4203cba00, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/generic-handlers.go:126 +0x60
minio1_1 | github.com/minio/minio/cmd.(*cacheControlHandler).ServeHTTP(0xc4203daf60, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:201 +0x79
minio1_1 | github.com/minio/minio/cmd.timeValidityHandler.ServeHTTP(0xede8c0, 0xc4203daf60, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/generic-handlers.go:215 +0x209
minio1_1 | github.com/minio/minio/cmd.(*timeValidityHandler).ServeHTTP(0xc4203daf70, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:203 +0x79
minio1_1 | github.com/minio/minio/vendor/github.com/rs/cors.(*Cors).Handler.func1(0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/vendor/github.com/rs/cors/cors.go:190 +0xe9
minio1_1 | net/http.HandlerFunc.ServeHTTP(0xc4203cba80, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /usr/local/go/src/net/http/server.go:1726 +0x44
minio1_1 | github.com/minio/minio/cmd.resourceHandler.ServeHTTP(0xedf040, 0xc4203cba80, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/generic-handlers.go:277 +0x115
minio1_1 | github.com/minio/minio/cmd.(*resourceHandler).ServeHTTP(0xc4203dafe0, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:204 +0x79
minio1_1 | github.com/minio/minio/cmd.authHandler.ServeHTTP(0xedea80, 0xc4203dafe0, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/auth-handler.go:237 +0x169
minio1_1 | github.com/minio/minio/cmd.(*authHandler).ServeHTTP(0xc4203daff0, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | <autogenerated>:73 +0x79
minio1_1 | github.com/minio/minio/cmd.(*ServerMux).ListenAndServe.func1(0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /go/src/github.com/minio/minio/cmd/server-mux.go:356 +0x87
minio1_1 | net/http.HandlerFunc.ServeHTTP(0xc420410180, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /usr/local/go/src/net/http/server.go:1726 +0x44
minio1_1 | net/http.serverHandler.ServeHTTP(0xc4203fb200, 0xee3180, 0xc427a61450, 0xc4202f6870)
minio1_1 | /usr/local/go/src/net/http/server.go:2202 +0x7d
minio1_1 | net/http.(*conn).serve(0xc420376d00, 0xee3bc0, 0xc427a6adc0)
minio1_1 | /usr/local/go/src/net/http/server.go:1579 +0x4b7
minio1_1 | created by net/http.(*Server).Serve
minio1_1 | /usr/local/go/src/net/http/server.go:2293 +0x44d